2 * linux/arch/x86_64/entry.S
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
6 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
10 * entry.S contains the system-call and fault low-level handling routines.
12 * NOTE: This code handles signal-recognition, which happens every time
13 * after an interrupt and after each system call.
15 * Normal syscalls and interrupts don't save a full stack frame, this is
16 * only done for syscall tracing, signals or fork/exec et.al.
18 * A note on terminology:
19 * - top of stack: Architecture defined interrupt frame from SS to RIP
20 * at the top of the kernel process stack.
21 * - partial stack frame: partially saved registers upto R11.
22 * - full stack frame: Like partial stack frame, but all register saved.
25 * - CFI macros are used to generate dwarf2 unwind information for better
26 * backtraces. They don't change any code.
27 * - SAVE_ALL/RESTORE_ALL - Save/restore all registers
28 * - SAVE_ARGS/RESTORE_ARGS - Save/restore registers that C functions modify.
29 * There are unfortunately lots of special cases where some registers
30 * not touched. The macro is a big mess that should be cleaned up.
31 * - SAVE_REST/RESTORE_REST - Handle the registers not saved by SAVE_ARGS.
32 * Gives a full stack frame.
33 * - ENTRY/END Define functions in the symbol table.
34 * - FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK - Fix up the hardware stack
35 * frame that is otherwise undefined after a SYSCALL
36 * - TRACE_IRQ_* - Trace hard interrupt state for lock debugging.
37 * - errorentry/paranoidentry/zeroentry - Define exception entry points.
40 #include <linux/linkage.h>
41 #include <asm/segment.h>
42 #include <asm/cache.h>
43 #include <asm/errno.h>
44 #include <asm/dwarf2.h>
45 #include <asm/calling.h>
46 #include <asm/asm-offsets.h>
48 #include <asm/unistd.h>
49 #include <asm/thread_info.h>
50 #include <asm/hw_irq.h>
52 #include <asm/irqflags.h>
53 #include <asm/paravirt.h>
54 #include <asm/ftrace.h>
59 #ifdef CONFIG_DYNAMIC_FTRACE
72 subq $MCOUNT_INSN_SIZE, %rdi
92 /* taken from glibc */
102 movq 0x38(%rsp), %rdi
104 subq $MCOUNT_INSN_SIZE, %rdi
124 #else /* ! CONFIG_DYNAMIC_FTRACE */
126 cmpq $ftrace_stub, ftrace_trace_function
133 /* taken from glibc */
143 movq 0x38(%rsp), %rdi
145 subq $MCOUNT_INSN_SIZE, %rdi
147 call *ftrace_trace_function
160 #endif /* CONFIG_DYNAMIC_FTRACE */
161 #endif /* CONFIG_FTRACE */
163 #ifndef CONFIG_PREEMPT
164 #define retint_kernel retint_restore_args
167 #ifdef CONFIG_PARAVIRT
168 ENTRY(native_irq_enable_syscall_ret)
169 movq %gs:pda_oldrsp,%rsp
172 #endif /* CONFIG_PARAVIRT */
175 .macro TRACE_IRQS_IRETQ offset=ARGOFFSET
176 #ifdef CONFIG_TRACE_IRQFLAGS
177 bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */
185 * C code is not supposed to know about undefined top of stack. Every time
186 * a C function with an pt_regs argument is called from the SYSCALL based
187 * fast path FIXUP_TOP_OF_STACK is needed.
188 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
192 /* %rsp:at FRAMEEND */
193 .macro FIXUP_TOP_OF_STACK tmp
194 movq %gs:pda_oldrsp,\tmp
196 movq $__USER_DS,SS(%rsp)
197 movq $__USER_CS,CS(%rsp)
199 movq R11(%rsp),\tmp /* get eflags */
200 movq \tmp,EFLAGS(%rsp)
203 .macro RESTORE_TOP_OF_STACK tmp,offset=0
204 movq RSP-\offset(%rsp),\tmp
205 movq \tmp,%gs:pda_oldrsp
206 movq EFLAGS-\offset(%rsp),\tmp
207 movq \tmp,R11-\offset(%rsp)
210 .macro FAKE_STACK_FRAME child_rip
211 /* push in order ss, rsp, eflags, cs, rip */
214 CFI_ADJUST_CFA_OFFSET 8
215 /*CFI_REL_OFFSET ss,0*/
217 CFI_ADJUST_CFA_OFFSET 8
219 pushq $(1<<9) /* eflags - interrupts on */
220 CFI_ADJUST_CFA_OFFSET 8
221 /*CFI_REL_OFFSET rflags,0*/
222 pushq $__KERNEL_CS /* cs */
223 CFI_ADJUST_CFA_OFFSET 8
224 /*CFI_REL_OFFSET cs,0*/
225 pushq \child_rip /* rip */
226 CFI_ADJUST_CFA_OFFSET 8
228 pushq %rax /* orig rax */
229 CFI_ADJUST_CFA_OFFSET 8
232 .macro UNFAKE_STACK_FRAME
234 CFI_ADJUST_CFA_OFFSET -(6*8)
237 .macro CFI_DEFAULT_STACK start=1
243 CFI_DEF_CFA_OFFSET SS+8
245 CFI_REL_OFFSET r15,R15
246 CFI_REL_OFFSET r14,R14
247 CFI_REL_OFFSET r13,R13
248 CFI_REL_OFFSET r12,R12
249 CFI_REL_OFFSET rbp,RBP
250 CFI_REL_OFFSET rbx,RBX
251 CFI_REL_OFFSET r11,R11
252 CFI_REL_OFFSET r10,R10
255 CFI_REL_OFFSET rax,RAX
256 CFI_REL_OFFSET rcx,RCX
257 CFI_REL_OFFSET rdx,RDX
258 CFI_REL_OFFSET rsi,RSI
259 CFI_REL_OFFSET rdi,RDI
260 CFI_REL_OFFSET rip,RIP
261 /*CFI_REL_OFFSET cs,CS*/
262 /*CFI_REL_OFFSET rflags,EFLAGS*/
263 CFI_REL_OFFSET rsp,RSP
264 /*CFI_REL_OFFSET ss,SS*/
267 * A newly forked process directly context switches into this.
272 push kernel_eflags(%rip)
273 CFI_ADJUST_CFA_OFFSET 4
274 popf # reset kernel eflags
275 CFI_ADJUST_CFA_OFFSET -4
277 GET_THREAD_INFO(%rcx)
278 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx)
282 testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?
283 je int_ret_from_sys_call
284 testl $_TIF_IA32,threadinfo_flags(%rcx)
285 jnz int_ret_from_sys_call
286 RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
287 jmp ret_from_sys_call
290 call syscall_trace_leave
291 GET_THREAD_INFO(%rcx)
297 * System call entry. Upto 6 arguments in registers are supported.
299 * SYSCALL does not save anything on the stack and does not change the
305 * rax system call number
307 * rcx return address for syscall/sysret, C arg3
310 * r10 arg3 (--> moved to rcx for C)
313 * r11 eflags for syscall/sysret, temporary for C
314 * r12-r15,rbp,rbx saved by C code, not touched.
316 * Interrupts are off on entry.
317 * Only called from user space.
319 * XXX if we had a free scratch register we could save the RSP into the stack frame
320 * and report it properly in ps. Unfortunately we haven't.
322 * When user can change the frames always force IRET. That is because
323 * it deals with uncanonical addresses better. SYSRET has trouble
324 * with them due to bugs in both AMD and Intel CPUs.
330 CFI_DEF_CFA rsp,PDA_STACKOFFSET
332 /*CFI_REGISTER rflags,r11*/
335 * A hypervisor implementation might want to use a label
336 * after the swapgs, so that it can do the swapgs
337 * for the guest and jump here on syscall.
339 ENTRY(system_call_after_swapgs)
341 movq %rsp,%gs:pda_oldrsp
342 movq %gs:pda_kernelstack,%rsp
344 * No need to follow this irqs off/on section - it's straight
347 ENABLE_INTERRUPTS(CLBR_NONE)
349 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
350 movq %rcx,RIP-ARGOFFSET(%rsp)
351 CFI_REL_OFFSET rip,RIP-ARGOFFSET
352 GET_THREAD_INFO(%rcx)
353 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
355 cmpq $__NR_syscall_max,%rax
358 call *sys_call_table(,%rax,8) # XXX: rip relative
359 movq %rax,RAX-ARGOFFSET(%rsp)
361 * Syscall return path ending with SYSRET (fast path)
362 * Has incomplete stack frame and undefined top of stack.
365 movl $_TIF_ALLWORK_MASK,%edi
369 GET_THREAD_INFO(%rcx)
370 DISABLE_INTERRUPTS(CLBR_NONE)
372 movl threadinfo_flags(%rcx),%edx
377 * sysretq will re-enable interrupts:
380 movq RIP-ARGOFFSET(%rsp),%rcx
382 RESTORE_ARGS 0,-ARG_SKIP,1
383 /*CFI_REGISTER rflags,r11*/
384 ENABLE_INTERRUPTS_SYSCALL_RET
387 /* Handle reschedules */
388 /* edx: work, edi: workmask */
390 bt $TIF_NEED_RESCHED,%edx
393 ENABLE_INTERRUPTS(CLBR_NONE)
395 CFI_ADJUST_CFA_OFFSET 8
398 CFI_ADJUST_CFA_OFFSET -8
401 /* Handle a signal */
404 ENABLE_INTERRUPTS(CLBR_NONE)
405 testl $_TIF_DO_NOTIFY_MASK,%edx
408 /* Really a signal */
409 /* edx: work flags (arg3) */
410 leaq do_notify_resume(%rip),%rax
411 leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
412 xorl %esi,%esi # oldset -> arg2
413 call ptregscall_common
414 1: movl $_TIF_NEED_RESCHED,%edi
415 /* Use IRET because user could have changed frame. This
416 works because ptregscall_common has called FIXUP_TOP_OF_STACK. */
417 DISABLE_INTERRUPTS(CLBR_NONE)
422 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
423 jmp ret_from_sys_call
425 /* Do syscall tracing */
428 movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
429 FIXUP_TOP_OF_STACK %rdi
431 call syscall_trace_enter
432 LOAD_ARGS ARGOFFSET /* reload args from stack in case ptrace changed it */
434 cmpq $__NR_syscall_max,%rax
435 ja int_ret_from_sys_call /* RAX(%rsp) set to -ENOSYS above */
436 movq %r10,%rcx /* fixup for C */
437 call *sys_call_table(,%rax,8)
438 movq %rax,RAX-ARGOFFSET(%rsp)
439 /* Use IRET because user could have changed frame */
442 * Syscall return path ending with IRET.
443 * Has correct top of stack, but partial stack frame.
445 .globl int_ret_from_sys_call
446 int_ret_from_sys_call:
447 DISABLE_INTERRUPTS(CLBR_NONE)
449 testl $3,CS-ARGOFFSET(%rsp)
450 je retint_restore_args
451 movl $_TIF_ALLWORK_MASK,%edi
452 /* edi: mask to check */
455 GET_THREAD_INFO(%rcx)
456 movl threadinfo_flags(%rcx),%edx
459 andl $~TS_COMPAT,threadinfo_status(%rcx)
462 /* Either reschedule or signal or syscall exit tracking needed. */
463 /* First do a reschedule test. */
464 /* edx: work, edi: workmask */
466 bt $TIF_NEED_RESCHED,%edx
469 ENABLE_INTERRUPTS(CLBR_NONE)
471 CFI_ADJUST_CFA_OFFSET 8
474 CFI_ADJUST_CFA_OFFSET -8
475 DISABLE_INTERRUPTS(CLBR_NONE)
479 /* handle signals and tracing -- both require a full stack frame */
482 ENABLE_INTERRUPTS(CLBR_NONE)
484 /* Check for syscall exit trace */
485 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
488 CFI_ADJUST_CFA_OFFSET 8
489 leaq 8(%rsp),%rdi # &ptregs -> arg1
490 call syscall_trace_leave
492 CFI_ADJUST_CFA_OFFSET -8
493 andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
497 testl $_TIF_DO_NOTIFY_MASK,%edx
499 movq %rsp,%rdi # &ptregs -> arg1
500 xorl %esi,%esi # oldset -> arg2
501 call do_notify_resume
502 1: movl $_TIF_NEED_RESCHED,%edi
505 DISABLE_INTERRUPTS(CLBR_NONE)
512 * Certain special system calls that need to save a complete full stack frame.
515 .macro PTREGSCALL label,func,arg
518 leaq \func(%rip),%rax
519 leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
520 jmp ptregscall_common
526 PTREGSCALL stub_clone, sys_clone, %r8
527 PTREGSCALL stub_fork, sys_fork, %rdi
528 PTREGSCALL stub_vfork, sys_vfork, %rdi
529 PTREGSCALL stub_rt_sigsuspend, sys_rt_sigsuspend, %rdx
530 PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
531 PTREGSCALL stub_iopl, sys_iopl, %rsi
533 ENTRY(ptregscall_common)
535 CFI_ADJUST_CFA_OFFSET -8
536 CFI_REGISTER rip, r11
539 CFI_REGISTER rip, r15
540 FIXUP_TOP_OF_STACK %r11
542 RESTORE_TOP_OF_STACK %r11
544 CFI_REGISTER rip, r11
547 CFI_ADJUST_CFA_OFFSET 8
548 CFI_REL_OFFSET rip, 0
551 END(ptregscall_common)
556 CFI_ADJUST_CFA_OFFSET -8
557 CFI_REGISTER rip, r11
559 FIXUP_TOP_OF_STACK %r11
562 RESTORE_TOP_OF_STACK %r11
565 jmp int_ret_from_sys_call
570 * sigreturn is special because it needs to restore all registers on return.
571 * This cannot be done with SYSRET, so use the IRET return path instead.
573 ENTRY(stub_rt_sigreturn)
576 CFI_ADJUST_CFA_OFFSET -8
579 FIXUP_TOP_OF_STACK %r11
580 call sys_rt_sigreturn
581 movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
583 jmp int_ret_from_sys_call
585 END(stub_rt_sigreturn)
588 * initial frame state for interrupts and exceptions
593 CFI_DEF_CFA rsp,SS+8-\ref
594 /*CFI_REL_OFFSET ss,SS-\ref*/
595 CFI_REL_OFFSET rsp,RSP-\ref
596 /*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
597 /*CFI_REL_OFFSET cs,CS-\ref*/
598 CFI_REL_OFFSET rip,RIP-\ref
601 /* initial frame state for interrupts (and exceptions without error code) */
602 #define INTR_FRAME _frame RIP
603 /* initial frame state for exceptions with error code (and interrupts with
604 vector already pushed) */
605 #define XCPT_FRAME _frame ORIG_RAX
608 * Interrupt entry/exit.
610 * Interrupt entry points save only callee clobbered registers in fast path.
612 * Entry runs with interrupts off.
615 /* 0(%rsp): interrupt number */
616 .macro interrupt func
619 leaq -ARGOFFSET(%rsp),%rdi # arg1 for handler
621 CFI_ADJUST_CFA_OFFSET 8
622 CFI_REL_OFFSET rbp, 0
624 CFI_DEF_CFA_REGISTER rbp
628 /* irqcount is used to check if a CPU is already on an interrupt
629 stack or not. While this is essentially redundant with preempt_count
630 it is a little cheaper to use a separate counter in the PDA
631 (short of moving irq_enter into assembly, which would be too
633 1: incl %gs:pda_irqcount
634 cmoveq %gs:pda_irqstackptr,%rsp
635 push %rbp # backlink for old unwinder
637 * We entered an interrupt context - irqs are off:
643 ENTRY(common_interrupt)
646 /* 0(%rsp): oldrsp-ARGOFFSET */
648 DISABLE_INTERRUPTS(CLBR_NONE)
650 decl %gs:pda_irqcount
652 CFI_DEF_CFA_REGISTER rsp
653 CFI_ADJUST_CFA_OFFSET -8
655 GET_THREAD_INFO(%rcx)
656 testl $3,CS-ARGOFFSET(%rsp)
659 /* Interrupt came from user space */
661 * Has a correct top of stack, but a partial stack frame
662 * %rcx: thread info. Interrupts off.
664 retint_with_reschedule:
665 movl $_TIF_WORK_MASK,%edi
668 movl threadinfo_flags(%rcx),%edx
673 retint_swapgs: /* return to user-space */
675 * The iretq could re-enable interrupts:
677 DISABLE_INTERRUPTS(CLBR_ANY)
682 retint_restore_args: /* return to kernel space */
683 DISABLE_INTERRUPTS(CLBR_ANY)
685 * The iretq could re-enable interrupts:
694 .section __ex_table, "a"
695 .quad irq_return, bad_iret
698 #ifdef CONFIG_PARAVIRT
702 .section __ex_table,"a"
703 .quad native_iret, bad_iret
710 * The iret traps when the %cs or %ss being restored is bogus.
711 * We've lost the original trap vector and error code.
712 * #GPF is the most likely one to get for an invalid selector.
713 * So pretend we completed the iret and took the #GPF in user mode.
715 * We are now running with the kernel GS after exception recovery.
716 * But error_entry expects us to have user GS to match the user %cs,
722 jmp general_protection
726 /* edi: workmask, edx: work */
729 bt $TIF_NEED_RESCHED,%edx
732 ENABLE_INTERRUPTS(CLBR_NONE)
734 CFI_ADJUST_CFA_OFFSET 8
737 CFI_ADJUST_CFA_OFFSET -8
738 GET_THREAD_INFO(%rcx)
739 DISABLE_INTERRUPTS(CLBR_NONE)
744 testl $_TIF_DO_NOTIFY_MASK,%edx
747 ENABLE_INTERRUPTS(CLBR_NONE)
749 movq $-1,ORIG_RAX(%rsp)
750 xorl %esi,%esi # oldset
751 movq %rsp,%rdi # &pt_regs
752 call do_notify_resume
754 DISABLE_INTERRUPTS(CLBR_NONE)
756 movl $_TIF_NEED_RESCHED,%edi
757 GET_THREAD_INFO(%rcx)
760 #ifdef CONFIG_PREEMPT
761 /* Returning to kernel space. Check if we need preemption */
762 /* rcx: threadinfo. interrupts off. */
764 cmpl $0,threadinfo_preempt_count(%rcx)
765 jnz retint_restore_args
766 bt $TIF_NEED_RESCHED,threadinfo_flags(%rcx)
767 jnc retint_restore_args
768 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
769 jnc retint_restore_args
770 call preempt_schedule_irq
775 END(common_interrupt)
780 .macro apicinterrupt num,func
783 CFI_ADJUST_CFA_OFFSET 8
789 ENTRY(thermal_interrupt)
790 apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
791 END(thermal_interrupt)
793 ENTRY(threshold_interrupt)
794 apicinterrupt THRESHOLD_APIC_VECTOR,mce_threshold_interrupt
795 END(threshold_interrupt)
798 ENTRY(reschedule_interrupt)
799 apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
800 END(reschedule_interrupt)
802 .macro INVALIDATE_ENTRY num
803 ENTRY(invalidate_interrupt\num)
804 apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt
805 END(invalidate_interrupt\num)
817 ENTRY(call_function_interrupt)
818 apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
819 END(call_function_interrupt)
820 ENTRY(irq_move_cleanup_interrupt)
821 apicinterrupt IRQ_MOVE_CLEANUP_VECTOR,smp_irq_move_cleanup_interrupt
822 END(irq_move_cleanup_interrupt)
825 ENTRY(apic_timer_interrupt)
826 apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
827 END(apic_timer_interrupt)
829 ENTRY(error_interrupt)
830 apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
833 ENTRY(spurious_interrupt)
834 apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
835 END(spurious_interrupt)
838 * Exception entry points.
842 pushq $0 /* push error code/oldrax */
843 CFI_ADJUST_CFA_OFFSET 8
844 pushq %rax /* push real oldrax to the rdi slot */
845 CFI_ADJUST_CFA_OFFSET 8
852 .macro errorentry sym
855 CFI_ADJUST_CFA_OFFSET 8
862 /* error code is on the stack already */
863 /* handle NMI like exceptions that can happen everywhere */
864 .macro paranoidentry sym, ist=0, irqtrace=1
868 movl $MSR_GS_BASE,%ecx
876 movq %gs:pda_data_offset, %rbp
879 movq ORIG_RAX(%rsp),%rsi
880 movq $-1,ORIG_RAX(%rsp)
882 subq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
886 addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
888 DISABLE_INTERRUPTS(CLBR_NONE)
895 * "Paranoid" exit path from exception stack.
896 * Paranoid because this is used by NMIs and cannot take
897 * any kernel state for granted.
898 * We don't do kernel preemption checks here, because only
899 * NMI should be common and it does not enable IRQs and
900 * cannot get reschedule ticks.
902 * "trace" is 0 for the NMI handler only, because irq-tracing
903 * is fundamentally NMI-unsafe. (we cannot change the soft and
904 * hard flags at once, atomically)
906 .macro paranoidexit trace=1
907 /* ebx: no swapgs flag */
909 testl %ebx,%ebx /* swapgs needed? */
910 jnz paranoid_restore\trace
912 jnz paranoid_userspace\trace
913 paranoid_swapgs\trace:
918 paranoid_restore\trace:
921 paranoid_userspace\trace:
922 GET_THREAD_INFO(%rcx)
923 movl threadinfo_flags(%rcx),%ebx
924 andl $_TIF_WORK_MASK,%ebx
925 jz paranoid_swapgs\trace
926 movq %rsp,%rdi /* &pt_regs */
928 movq %rax,%rsp /* switch stack for scheduling */
929 testl $_TIF_NEED_RESCHED,%ebx
930 jnz paranoid_schedule\trace
931 movl %ebx,%edx /* arg3: thread flags */
935 ENABLE_INTERRUPTS(CLBR_NONE)
936 xorl %esi,%esi /* arg2: oldset */
937 movq %rsp,%rdi /* arg1: &pt_regs */
938 call do_notify_resume
939 DISABLE_INTERRUPTS(CLBR_NONE)
943 jmp paranoid_userspace\trace
944 paranoid_schedule\trace:
948 ENABLE_INTERRUPTS(CLBR_ANY)
950 DISABLE_INTERRUPTS(CLBR_ANY)
954 jmp paranoid_userspace\trace
959 * Exception entry point. This expects an error code/orig_rax on the stack
960 * and the exception handler in %rax.
962 KPROBE_ENTRY(error_entry)
965 /* rdi slot contains rax, oldrax contains error code */
968 CFI_ADJUST_CFA_OFFSET (14*8)
970 CFI_REL_OFFSET rsi,RSI
971 movq 14*8(%rsp),%rsi /* load rax from rdi slot */
974 CFI_REL_OFFSET rdx,RDX
976 CFI_REL_OFFSET rcx,RCX
977 movq %rsi,10*8(%rsp) /* store rax */
978 CFI_REL_OFFSET rax,RAX
984 CFI_REL_OFFSET r10,R10
986 CFI_REL_OFFSET r11,R11
988 CFI_REL_OFFSET rbx,RBX
990 CFI_REL_OFFSET rbp,RBP
992 CFI_REL_OFFSET r12,R12
994 CFI_REL_OFFSET r13,R13
996 CFI_REL_OFFSET r14,R14
998 CFI_REL_OFFSET r15,R15
1001 je error_kernelspace
1006 CFI_REL_OFFSET rdi,RDI
1008 movq ORIG_RAX(%rsp),%rsi /* get error code */
1009 movq $-1,ORIG_RAX(%rsp)
1011 /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
1015 DISABLE_INTERRUPTS(CLBR_NONE)
1017 GET_THREAD_INFO(%rcx)
1020 LOCKDEP_SYS_EXIT_IRQ
1021 movl threadinfo_flags(%rcx),%edx
1022 movl $_TIF_WORK_MASK,%edi
1030 /* There are two places in the kernel that can potentially fault with
1031 usergs. Handle them here. The exception handlers after
1032 iret run with kernel gs again, so don't set the user space flag.
1033 B stepping K8s sometimes report an truncated RIP for IRET
1034 exceptions returning to compat mode. Check for these here too. */
1035 leaq irq_return(%rip),%rbp
1038 movl %ebp,%ebp /* zero extend */
1041 cmpq $gs_change,RIP(%rsp)
1044 KPROBE_END(error_entry)
1046 /* Reload gs selector with exception handling */
1047 /* edi: new selector */
1048 ENTRY(load_gs_index)
1051 CFI_ADJUST_CFA_OFFSET 8
1052 DISABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI))
1056 2: mfence /* workaround */
1059 CFI_ADJUST_CFA_OFFSET -8
1062 ENDPROC(load_gs_index)
1064 .section __ex_table,"a"
1066 .quad gs_change,bad_gs
1068 .section .fixup,"ax"
1069 /* running with kernelgs */
1071 SWAPGS /* switch back to user gs */
1078 * Create a kernel thread.
1080 * C extern interface:
1081 * extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
1083 * asm input arguments:
1084 * rdi: fn, rsi: arg, rdx: flags
1086 ENTRY(kernel_thread)
1088 FAKE_STACK_FRAME $child_rip
1091 # rdi: flags, rsi: usp, rdx: will be &pt_regs
1093 orq kernel_thread_flags(%rip),%rdi
1106 * It isn't worth to check for reschedule here,
1107 * so internally to the x86_64 port you can rely on kernel_thread()
1108 * not to reschedule the child before returning, this avoids the need
1109 * of hacks for example to fork off the per-CPU idle tasks.
1110 * [Hopefully no generic code relies on the reschedule -AK]
1116 ENDPROC(kernel_thread)
1119 pushq $0 # fake return address
1122 * Here we are in the child and the registers are set as they were
1123 * at kernel_thread() invocation in the parent.
1135 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
1137 * C extern interface:
1138 * extern long execve(char *name, char **argv, char **envp)
1140 * asm input arguments:
1141 * rdi: name, rsi: argv, rdx: envp
1143 * We want to fallback into:
1144 * extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs *regs)
1146 * do_sys_execve asm fallback arguments:
1147 * rdi: name, rsi: argv, rdx: envp, rcx: fake frame on the stack
1149 ENTRY(kernel_execve)
1155 movq %rax, RAX(%rsp)
1158 je int_ret_from_sys_call
1163 ENDPROC(kernel_execve)
1165 KPROBE_ENTRY(page_fault)
1166 errorentry do_page_fault
1167 KPROBE_END(page_fault)
1169 ENTRY(coprocessor_error)
1170 zeroentry do_coprocessor_error
1171 END(coprocessor_error)
1173 ENTRY(simd_coprocessor_error)
1174 zeroentry do_simd_coprocessor_error
1175 END(simd_coprocessor_error)
1177 ENTRY(device_not_available)
1178 zeroentry math_state_restore
1179 END(device_not_available)
1181 /* runs on exception stack */
1185 CFI_ADJUST_CFA_OFFSET 8
1186 paranoidentry do_debug, DEBUG_STACK
1190 /* runs on exception stack */
1194 CFI_ADJUST_CFA_OFFSET 8
1195 paranoidentry do_nmi, 0, 0
1196 #ifdef CONFIG_TRACE_IRQFLAGS
1207 CFI_ADJUST_CFA_OFFSET 8
1208 paranoidentry do_int3, DEBUG_STACK
1214 zeroentry do_overflow
1222 zeroentry do_invalid_op
1225 ENTRY(coprocessor_segment_overrun)
1226 zeroentry do_coprocessor_segment_overrun
1227 END(coprocessor_segment_overrun)
1230 zeroentry do_reserved
1233 /* runs on exception stack */
1236 paranoidentry do_double_fault
1242 errorentry do_invalid_TSS
1245 ENTRY(segment_not_present)
1246 errorentry do_segment_not_present
1247 END(segment_not_present)
1249 /* runs on exception stack */
1250 ENTRY(stack_segment)
1252 paranoidentry do_stack_segment
1257 KPROBE_ENTRY(general_protection)
1258 errorentry do_general_protection
1259 KPROBE_END(general_protection)
1261 ENTRY(alignment_check)
1262 errorentry do_alignment_check
1263 END(alignment_check)
1266 zeroentry do_divide_error
1269 ENTRY(spurious_interrupt_bug)
1270 zeroentry do_spurious_interrupt_bug
1271 END(spurious_interrupt_bug)
1273 #ifdef CONFIG_X86_MCE
1274 /* runs on exception stack */
1275 ENTRY(machine_check)
1278 CFI_ADJUST_CFA_OFFSET 8
1279 paranoidentry do_machine_check
1285 /* Call softirq on interrupt stack. Interrupts are off. */
1289 CFI_ADJUST_CFA_OFFSET 8
1290 CFI_REL_OFFSET rbp,0
1292 CFI_DEF_CFA_REGISTER rbp
1293 incl %gs:pda_irqcount
1294 cmove %gs:pda_irqstackptr,%rsp
1295 push %rbp # backlink for old unwinder
1298 CFI_DEF_CFA_REGISTER rsp
1299 CFI_ADJUST_CFA_OFFSET -8
1300 decl %gs:pda_irqcount
1303 ENDPROC(call_softirq)
1305 KPROBE_ENTRY(ignore_sysret)
1310 ENDPROC(ignore_sysret)