2 * linux/arch/x86_64/entry.S
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
6 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
10 * entry.S contains the system-call and fault low-level handling routines.
12 * NOTE: This code handles signal-recognition, which happens every time
13 * after an interrupt and after each system call.
15 * Normal syscalls and interrupts don't save a full stack frame, this is
16 * only done for syscall tracing, signals or fork/exec et.al.
18 * A note on terminology:
19 * - top of stack: Architecture defined interrupt frame from SS to RIP
20 * at the top of the kernel process stack.
21 * - partial stack frame: partially saved registers upto R11.
22 * - full stack frame: Like partial stack frame, but all register saved.
25 * - CFI macros are used to generate dwarf2 unwind information for better
26 * backtraces. They don't change any code.
27 * - SAVE_ALL/RESTORE_ALL - Save/restore all registers
28 * - SAVE_ARGS/RESTORE_ARGS - Save/restore registers that C functions modify.
29 * There are unfortunately lots of special cases where some registers
30 * not touched. The macro is a big mess that should be cleaned up.
31 * - SAVE_REST/RESTORE_REST - Handle the registers not saved by SAVE_ARGS.
32 * Gives a full stack frame.
33 * - ENTRY/END Define functions in the symbol table.
34 * - FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK - Fix up the hardware stack
35 * frame that is otherwise undefined after a SYSCALL
36 * - TRACE_IRQ_* - Trace hard interrupt state for lock debugging.
37 * - errorentry/paranoidentry/zeroentry - Define exception entry points.
40 #include <linux/linkage.h>
41 #include <asm/segment.h>
42 #include <asm/cache.h>
43 #include <asm/errno.h>
44 #include <asm/dwarf2.h>
45 #include <asm/calling.h>
46 #include <asm/asm-offsets.h>
48 #include <asm/unistd.h>
49 #include <asm/thread_info.h>
50 #include <asm/hw_irq.h>
52 #include <asm/irqflags.h>
53 #include <asm/paravirt.h>
54 #include <asm/ftrace.h>
56 /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */
57 #include <linux/elf-em.h>
58 #define AUDIT_ARCH_X86_64 (EM_X86_64|__AUDIT_ARCH_64BIT|__AUDIT_ARCH_LE)
59 #define __AUDIT_ARCH_64BIT 0x80000000
60 #define __AUDIT_ARCH_LE 0x40000000
64 #ifdef CONFIG_FUNCTION_TRACER
65 #ifdef CONFIG_DYNAMIC_FTRACE
72 /* taken from glibc */
84 subq $MCOUNT_INSN_SIZE, %rdi
104 #else /* ! CONFIG_DYNAMIC_FTRACE */
106 cmpq $ftrace_stub, ftrace_trace_function
113 /* taken from glibc */
123 movq 0x38(%rsp), %rdi
125 subq $MCOUNT_INSN_SIZE, %rdi
127 call *ftrace_trace_function
140 #endif /* CONFIG_DYNAMIC_FTRACE */
141 #endif /* CONFIG_FUNCTION_TRACER */
143 #ifndef CONFIG_PREEMPT
144 #define retint_kernel retint_restore_args
147 #ifdef CONFIG_PARAVIRT
148 ENTRY(native_usergs_sysret64)
151 #endif /* CONFIG_PARAVIRT */
154 .macro TRACE_IRQS_IRETQ offset=ARGOFFSET
155 #ifdef CONFIG_TRACE_IRQFLAGS
156 bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */
164 * C code is not supposed to know about undefined top of stack. Every time
165 * a C function with an pt_regs argument is called from the SYSCALL based
166 * fast path FIXUP_TOP_OF_STACK is needed.
167 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
171 /* %rsp:at FRAMEEND */
172 .macro FIXUP_TOP_OF_STACK tmp
173 movq %gs:pda_oldrsp,\tmp
175 movq $__USER_DS,SS(%rsp)
176 movq $__USER_CS,CS(%rsp)
178 movq R11(%rsp),\tmp /* get eflags */
179 movq \tmp,EFLAGS(%rsp)
182 .macro RESTORE_TOP_OF_STACK tmp,offset=0
183 movq RSP-\offset(%rsp),\tmp
184 movq \tmp,%gs:pda_oldrsp
185 movq EFLAGS-\offset(%rsp),\tmp
186 movq \tmp,R11-\offset(%rsp)
189 .macro FAKE_STACK_FRAME child_rip
190 /* push in order ss, rsp, eflags, cs, rip */
192 pushq $__KERNEL_DS /* ss */
193 CFI_ADJUST_CFA_OFFSET 8
194 /*CFI_REL_OFFSET ss,0*/
196 CFI_ADJUST_CFA_OFFSET 8
198 pushq $(1<<9) /* eflags - interrupts on */
199 CFI_ADJUST_CFA_OFFSET 8
200 /*CFI_REL_OFFSET rflags,0*/
201 pushq $__KERNEL_CS /* cs */
202 CFI_ADJUST_CFA_OFFSET 8
203 /*CFI_REL_OFFSET cs,0*/
204 pushq \child_rip /* rip */
205 CFI_ADJUST_CFA_OFFSET 8
207 pushq %rax /* orig rax */
208 CFI_ADJUST_CFA_OFFSET 8
211 .macro UNFAKE_STACK_FRAME
213 CFI_ADJUST_CFA_OFFSET -(6*8)
216 .macro CFI_DEFAULT_STACK start=1
222 CFI_DEF_CFA_OFFSET SS+8
224 CFI_REL_OFFSET r15,R15
225 CFI_REL_OFFSET r14,R14
226 CFI_REL_OFFSET r13,R13
227 CFI_REL_OFFSET r12,R12
228 CFI_REL_OFFSET rbp,RBP
229 CFI_REL_OFFSET rbx,RBX
230 CFI_REL_OFFSET r11,R11
231 CFI_REL_OFFSET r10,R10
234 CFI_REL_OFFSET rax,RAX
235 CFI_REL_OFFSET rcx,RCX
236 CFI_REL_OFFSET rdx,RDX
237 CFI_REL_OFFSET rsi,RSI
238 CFI_REL_OFFSET rdi,RDI
239 CFI_REL_OFFSET rip,RIP
240 /*CFI_REL_OFFSET cs,CS*/
241 /*CFI_REL_OFFSET rflags,EFLAGS*/
242 CFI_REL_OFFSET rsp,RSP
243 /*CFI_REL_OFFSET ss,SS*/
246 * A newly forked process directly context switches into this.
251 push kernel_eflags(%rip)
252 CFI_ADJUST_CFA_OFFSET 8
253 popf # reset kernel eflags
254 CFI_ADJUST_CFA_OFFSET -8
256 GET_THREAD_INFO(%rcx)
257 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),TI_flags(%rcx)
261 testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?
262 je int_ret_from_sys_call
263 testl $_TIF_IA32,TI_flags(%rcx)
264 jnz int_ret_from_sys_call
265 RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
266 jmp ret_from_sys_call
269 call syscall_trace_leave
270 GET_THREAD_INFO(%rcx)
276 * System call entry. Upto 6 arguments in registers are supported.
278 * SYSCALL does not save anything on the stack and does not change the
284 * rax system call number
286 * rcx return address for syscall/sysret, C arg3
289 * r10 arg3 (--> moved to rcx for C)
292 * r11 eflags for syscall/sysret, temporary for C
293 * r12-r15,rbp,rbx saved by C code, not touched.
295 * Interrupts are off on entry.
296 * Only called from user space.
298 * XXX if we had a free scratch register we could save the RSP into the stack frame
299 * and report it properly in ps. Unfortunately we haven't.
301 * When user can change the frames always force IRET. That is because
302 * it deals with uncanonical addresses better. SYSRET has trouble
303 * with them due to bugs in both AMD and Intel CPUs.
309 CFI_DEF_CFA rsp,PDA_STACKOFFSET
311 /*CFI_REGISTER rflags,r11*/
314 * A hypervisor implementation might want to use a label
315 * after the swapgs, so that it can do the swapgs
316 * for the guest and jump here on syscall.
318 ENTRY(system_call_after_swapgs)
320 movq %rsp,%gs:pda_oldrsp
321 movq %gs:pda_kernelstack,%rsp
323 * No need to follow this irqs off/on section - it's straight
326 ENABLE_INTERRUPTS(CLBR_NONE)
328 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
329 movq %rcx,RIP-ARGOFFSET(%rsp)
330 CFI_REL_OFFSET rip,RIP-ARGOFFSET
331 GET_THREAD_INFO(%rcx)
332 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%rcx)
334 system_call_fastpath:
335 cmpq $__NR_syscall_max,%rax
338 call *sys_call_table(,%rax,8) # XXX: rip relative
339 movq %rax,RAX-ARGOFFSET(%rsp)
341 * Syscall return path ending with SYSRET (fast path)
342 * Has incomplete stack frame and undefined top of stack.
345 movl $_TIF_ALLWORK_MASK,%edi
349 GET_THREAD_INFO(%rcx)
350 DISABLE_INTERRUPTS(CLBR_NONE)
352 movl TI_flags(%rcx),%edx
357 * sysretq will re-enable interrupts:
360 movq RIP-ARGOFFSET(%rsp),%rcx
362 RESTORE_ARGS 0,-ARG_SKIP,1
363 /*CFI_REGISTER rflags,r11*/
364 movq %gs:pda_oldrsp, %rsp
368 /* Handle reschedules */
369 /* edx: work, edi: workmask */
371 bt $TIF_NEED_RESCHED,%edx
374 ENABLE_INTERRUPTS(CLBR_NONE)
376 CFI_ADJUST_CFA_OFFSET 8
379 CFI_ADJUST_CFA_OFFSET -8
382 /* Handle a signal */
385 ENABLE_INTERRUPTS(CLBR_NONE)
386 #ifdef CONFIG_AUDITSYSCALL
387 bt $TIF_SYSCALL_AUDIT,%edx
390 /* edx: work flags (arg3) */
391 leaq do_notify_resume(%rip),%rax
392 leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
393 xorl %esi,%esi # oldset -> arg2
394 call ptregscall_common
395 movl $_TIF_WORK_MASK,%edi
396 /* Use IRET because user could have changed frame. This
397 works because ptregscall_common has called FIXUP_TOP_OF_STACK. */
398 DISABLE_INTERRUPTS(CLBR_NONE)
403 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
404 jmp ret_from_sys_call
406 #ifdef CONFIG_AUDITSYSCALL
408 * Fast path for syscall audit without full syscall trace.
409 * We just call audit_syscall_entry() directly, and then
410 * jump back to the normal fast path.
413 movq %r10,%r9 /* 6th arg: 4th syscall arg */
414 movq %rdx,%r8 /* 5th arg: 3rd syscall arg */
415 movq %rsi,%rcx /* 4th arg: 2nd syscall arg */
416 movq %rdi,%rdx /* 3rd arg: 1st syscall arg */
417 movq %rax,%rsi /* 2nd arg: syscall number */
418 movl $AUDIT_ARCH_X86_64,%edi /* 1st arg: audit arch */
419 call audit_syscall_entry
420 LOAD_ARGS 0 /* reload call-clobbered registers */
421 jmp system_call_fastpath
424 * Return fast path for syscall audit. Call audit_syscall_exit()
425 * directly and then jump back to the fast path with TIF_SYSCALL_AUDIT
429 movq %rax,%rsi /* second arg, syscall return value */
430 cmpq $0,%rax /* is it < 0? */
431 setl %al /* 1 if so, 0 if not */
432 movzbl %al,%edi /* zero-extend that into %edi */
433 inc %edi /* first arg, 0->1(AUDITSC_SUCCESS), 1->2(AUDITSC_FAILURE) */
434 call audit_syscall_exit
435 movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
437 #endif /* CONFIG_AUDITSYSCALL */
439 /* Do syscall tracing */
441 #ifdef CONFIG_AUDITSYSCALL
442 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%rcx)
446 movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
447 FIXUP_TOP_OF_STACK %rdi
449 call syscall_trace_enter
451 * Reload arg registers from stack in case ptrace changed them.
452 * We don't reload %rax because syscall_trace_enter() returned
453 * the value it wants us to use in the table lookup.
455 LOAD_ARGS ARGOFFSET, 1
457 cmpq $__NR_syscall_max,%rax
458 ja int_ret_from_sys_call /* RAX(%rsp) set to -ENOSYS above */
459 movq %r10,%rcx /* fixup for C */
460 call *sys_call_table(,%rax,8)
461 movq %rax,RAX-ARGOFFSET(%rsp)
462 /* Use IRET because user could have changed frame */
465 * Syscall return path ending with IRET.
466 * Has correct top of stack, but partial stack frame.
468 .globl int_ret_from_sys_call
469 .globl int_with_check
470 int_ret_from_sys_call:
471 DISABLE_INTERRUPTS(CLBR_NONE)
473 testl $3,CS-ARGOFFSET(%rsp)
474 je retint_restore_args
475 movl $_TIF_ALLWORK_MASK,%edi
476 /* edi: mask to check */
479 GET_THREAD_INFO(%rcx)
480 movl TI_flags(%rcx),%edx
483 andl $~TS_COMPAT,TI_status(%rcx)
486 /* Either reschedule or signal or syscall exit tracking needed. */
487 /* First do a reschedule test. */
488 /* edx: work, edi: workmask */
490 bt $TIF_NEED_RESCHED,%edx
493 ENABLE_INTERRUPTS(CLBR_NONE)
495 CFI_ADJUST_CFA_OFFSET 8
498 CFI_ADJUST_CFA_OFFSET -8
499 DISABLE_INTERRUPTS(CLBR_NONE)
503 /* handle signals and tracing -- both require a full stack frame */
506 ENABLE_INTERRUPTS(CLBR_NONE)
508 /* Check for syscall exit trace */
509 testl $_TIF_WORK_SYSCALL_EXIT,%edx
512 CFI_ADJUST_CFA_OFFSET 8
513 leaq 8(%rsp),%rdi # &ptregs -> arg1
514 call syscall_trace_leave
516 CFI_ADJUST_CFA_OFFSET -8
517 andl $~(_TIF_WORK_SYSCALL_EXIT|_TIF_SYSCALL_EMU),%edi
521 testl $_TIF_DO_NOTIFY_MASK,%edx
523 movq %rsp,%rdi # &ptregs -> arg1
524 xorl %esi,%esi # oldset -> arg2
525 call do_notify_resume
526 1: movl $_TIF_WORK_MASK,%edi
529 DISABLE_INTERRUPTS(CLBR_NONE)
536 * Certain special system calls that need to save a complete full stack frame.
539 .macro PTREGSCALL label,func,arg
542 leaq \func(%rip),%rax
543 leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
544 jmp ptregscall_common
550 PTREGSCALL stub_clone, sys_clone, %r8
551 PTREGSCALL stub_fork, sys_fork, %rdi
552 PTREGSCALL stub_vfork, sys_vfork, %rdi
553 PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
554 PTREGSCALL stub_iopl, sys_iopl, %rsi
556 ENTRY(ptregscall_common)
558 CFI_ADJUST_CFA_OFFSET -8
559 CFI_REGISTER rip, r11
562 CFI_REGISTER rip, r15
563 FIXUP_TOP_OF_STACK %r11
565 RESTORE_TOP_OF_STACK %r11
567 CFI_REGISTER rip, r11
570 CFI_ADJUST_CFA_OFFSET 8
571 CFI_REL_OFFSET rip, 0
574 END(ptregscall_common)
579 CFI_ADJUST_CFA_OFFSET -8
580 CFI_REGISTER rip, r11
582 FIXUP_TOP_OF_STACK %r11
585 RESTORE_TOP_OF_STACK %r11
588 jmp int_ret_from_sys_call
593 * sigreturn is special because it needs to restore all registers on return.
594 * This cannot be done with SYSRET, so use the IRET return path instead.
596 ENTRY(stub_rt_sigreturn)
599 CFI_ADJUST_CFA_OFFSET -8
602 FIXUP_TOP_OF_STACK %r11
603 call sys_rt_sigreturn
604 movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
606 jmp int_ret_from_sys_call
608 END(stub_rt_sigreturn)
611 * initial frame state for interrupts and exceptions
616 CFI_DEF_CFA rsp,SS+8-\ref
617 /*CFI_REL_OFFSET ss,SS-\ref*/
618 CFI_REL_OFFSET rsp,RSP-\ref
619 /*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
620 /*CFI_REL_OFFSET cs,CS-\ref*/
621 CFI_REL_OFFSET rip,RIP-\ref
624 /* initial frame state for interrupts (and exceptions without error code) */
625 #define INTR_FRAME _frame RIP
626 /* initial frame state for exceptions with error code (and interrupts with
627 vector already pushed) */
628 #define XCPT_FRAME _frame ORIG_RAX
631 * Build the entry stubs and pointer table with some assembler magic.
632 * We pack 7 stubs into a single 32-byte chunk, which will fit in a
633 * single cache line on all modern x86 implementations.
635 .section .init.rodata,"a"
639 .p2align CONFIG_X86_L1_CACHE_SHIFT
640 ENTRY(irq_entries_start)
642 vector=FIRST_EXTERNAL_VECTOR
643 .rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7
646 .if vector < NR_VECTORS
647 .if vector <> FIRST_EXTERNAL_VECTOR
648 CFI_ADJUST_CFA_OFFSET -8
650 1: pushq $(~vector+0x80) /* Note: always in signed byte range */
651 CFI_ADJUST_CFA_OFFSET 8
652 .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6
661 2: jmp common_interrupt
664 END(irq_entries_start)
671 * Interrupt entry/exit.
673 * Interrupt entry points save only callee clobbered registers in fast path.
675 * Entry runs with interrupts off.
678 /* 0(%rsp): ~(interrupt number) */
679 .macro interrupt func
682 leaq -ARGOFFSET(%rsp),%rdi /* arg1 for handler */
685 * Save rbp twice: One is for marking the stack frame, as usual, and the
686 * other, to fill pt_regs properly. This is because bx comes right
687 * before the last saved register in that structure, and not bp. If the
688 * base pointer were in the place bx is today, this would not be needed.
691 CFI_ADJUST_CFA_OFFSET 8
692 CFI_REL_OFFSET rbp, 0
694 CFI_DEF_CFA_REGISTER rbp
698 /* irqcount is used to check if a CPU is already on an interrupt
699 stack or not. While this is essentially redundant with preempt_count
700 it is a little cheaper to use a separate counter in the PDA
701 (short of moving irq_enter into assembly, which would be too
703 1: incl %gs:pda_irqcount
704 cmoveq %gs:pda_irqstackptr,%rsp
705 push %rbp # backlink for old unwinder
707 * We entered an interrupt context - irqs are off:
714 * The interrupt stubs push (~vector+0x80) onto the stack and
715 * then jump to common_interrupt.
717 .p2align CONFIG_X86_L1_CACHE_SHIFT
720 addq $-0x80,(%rsp) /* Adjust vector to [-256,-1] range */
722 /* 0(%rsp): oldrsp-ARGOFFSET */
724 DISABLE_INTERRUPTS(CLBR_NONE)
726 decl %gs:pda_irqcount
728 CFI_DEF_CFA_REGISTER rsp
729 CFI_ADJUST_CFA_OFFSET -8
731 GET_THREAD_INFO(%rcx)
732 testl $3,CS-ARGOFFSET(%rsp)
735 /* Interrupt came from user space */
737 * Has a correct top of stack, but a partial stack frame
738 * %rcx: thread info. Interrupts off.
740 retint_with_reschedule:
741 movl $_TIF_WORK_MASK,%edi
744 movl TI_flags(%rcx),%edx
749 retint_swapgs: /* return to user-space */
751 * The iretq could re-enable interrupts:
753 DISABLE_INTERRUPTS(CLBR_ANY)
758 retint_restore_args: /* return to kernel space */
759 DISABLE_INTERRUPTS(CLBR_ANY)
761 * The iretq could re-enable interrupts:
770 .section __ex_table, "a"
771 .quad irq_return, bad_iret
774 #ifdef CONFIG_PARAVIRT
778 .section __ex_table,"a"
779 .quad native_iret, bad_iret
786 * The iret traps when the %cs or %ss being restored is bogus.
787 * We've lost the original trap vector and error code.
788 * #GPF is the most likely one to get for an invalid selector.
789 * So pretend we completed the iret and took the #GPF in user mode.
791 * We are now running with the kernel GS after exception recovery.
792 * But error_entry expects us to have user GS to match the user %cs,
798 jmp general_protection
802 /* edi: workmask, edx: work */
805 bt $TIF_NEED_RESCHED,%edx
808 ENABLE_INTERRUPTS(CLBR_NONE)
810 CFI_ADJUST_CFA_OFFSET 8
813 CFI_ADJUST_CFA_OFFSET -8
814 GET_THREAD_INFO(%rcx)
815 DISABLE_INTERRUPTS(CLBR_NONE)
820 testl $_TIF_DO_NOTIFY_MASK,%edx
823 ENABLE_INTERRUPTS(CLBR_NONE)
825 movq $-1,ORIG_RAX(%rsp)
826 xorl %esi,%esi # oldset
827 movq %rsp,%rdi # &pt_regs
828 call do_notify_resume
830 DISABLE_INTERRUPTS(CLBR_NONE)
832 GET_THREAD_INFO(%rcx)
833 jmp retint_with_reschedule
835 #ifdef CONFIG_PREEMPT
836 /* Returning to kernel space. Check if we need preemption */
837 /* rcx: threadinfo. interrupts off. */
839 cmpl $0,TI_preempt_count(%rcx)
840 jnz retint_restore_args
841 bt $TIF_NEED_RESCHED,TI_flags(%rcx)
842 jnc retint_restore_args
843 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
844 jnc retint_restore_args
845 call preempt_schedule_irq
850 END(common_interrupt)
855 .macro apicinterrupt num,func
858 CFI_ADJUST_CFA_OFFSET 8
864 ENTRY(thermal_interrupt)
865 apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
866 END(thermal_interrupt)
868 ENTRY(threshold_interrupt)
869 apicinterrupt THRESHOLD_APIC_VECTOR,mce_threshold_interrupt
870 END(threshold_interrupt)
873 ENTRY(reschedule_interrupt)
874 apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
875 END(reschedule_interrupt)
877 .macro INVALIDATE_ENTRY num
878 ENTRY(invalidate_interrupt\num)
879 apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt
880 END(invalidate_interrupt\num)
892 ENTRY(call_function_interrupt)
893 apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
894 END(call_function_interrupt)
895 ENTRY(call_function_single_interrupt)
896 apicinterrupt CALL_FUNCTION_SINGLE_VECTOR,smp_call_function_single_interrupt
897 END(call_function_single_interrupt)
898 ENTRY(irq_move_cleanup_interrupt)
899 apicinterrupt IRQ_MOVE_CLEANUP_VECTOR,smp_irq_move_cleanup_interrupt
900 END(irq_move_cleanup_interrupt)
903 ENTRY(apic_timer_interrupt)
904 apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
905 END(apic_timer_interrupt)
907 ENTRY(uv_bau_message_intr1)
908 apicinterrupt 220,uv_bau_message_interrupt
909 END(uv_bau_message_intr1)
911 ENTRY(error_interrupt)
912 apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
915 ENTRY(spurious_interrupt)
916 apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
917 END(spurious_interrupt)
920 * Exception entry points.
924 PARAVIRT_ADJUST_EXCEPTION_FRAME
925 pushq $0 /* push error code/oldrax */
926 CFI_ADJUST_CFA_OFFSET 8
927 pushq %rax /* push real oldrax to the rdi slot */
928 CFI_ADJUST_CFA_OFFSET 8
935 .macro errorentry sym
937 PARAVIRT_ADJUST_EXCEPTION_FRAME
939 CFI_ADJUST_CFA_OFFSET 8
946 /* error code is on the stack already */
947 /* handle NMI like exceptions that can happen everywhere */
948 .macro paranoidentry sym, ist=0, irqtrace=1
952 movl $MSR_GS_BASE,%ecx
960 movq %gs:pda_data_offset, %rbp
966 movq ORIG_RAX(%rsp),%rsi
967 movq $-1,ORIG_RAX(%rsp)
969 subq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
973 addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
975 DISABLE_INTERRUPTS(CLBR_NONE)
982 * "Paranoid" exit path from exception stack.
983 * Paranoid because this is used by NMIs and cannot take
984 * any kernel state for granted.
985 * We don't do kernel preemption checks here, because only
986 * NMI should be common and it does not enable IRQs and
987 * cannot get reschedule ticks.
989 * "trace" is 0 for the NMI handler only, because irq-tracing
990 * is fundamentally NMI-unsafe. (we cannot change the soft and
991 * hard flags at once, atomically)
993 .macro paranoidexit trace=1
994 /* ebx: no swapgs flag */
996 testl %ebx,%ebx /* swapgs needed? */
997 jnz paranoid_restore\trace
999 jnz paranoid_userspace\trace
1000 paranoid_swapgs\trace:
1005 paranoid_restore\trace:
1008 paranoid_userspace\trace:
1009 GET_THREAD_INFO(%rcx)
1010 movl TI_flags(%rcx),%ebx
1011 andl $_TIF_WORK_MASK,%ebx
1012 jz paranoid_swapgs\trace
1013 movq %rsp,%rdi /* &pt_regs */
1015 movq %rax,%rsp /* switch stack for scheduling */
1016 testl $_TIF_NEED_RESCHED,%ebx
1017 jnz paranoid_schedule\trace
1018 movl %ebx,%edx /* arg3: thread flags */
1022 ENABLE_INTERRUPTS(CLBR_NONE)
1023 xorl %esi,%esi /* arg2: oldset */
1024 movq %rsp,%rdi /* arg1: &pt_regs */
1025 call do_notify_resume
1026 DISABLE_INTERRUPTS(CLBR_NONE)
1030 jmp paranoid_userspace\trace
1031 paranoid_schedule\trace:
1035 ENABLE_INTERRUPTS(CLBR_ANY)
1037 DISABLE_INTERRUPTS(CLBR_ANY)
1041 jmp paranoid_userspace\trace
1046 * Exception entry point. This expects an error code/orig_rax on the stack
1047 * and the exception handler in %rax.
1049 KPROBE_ENTRY(error_entry)
1051 CFI_REL_OFFSET rax,0
1052 /* rdi slot contains rax, oldrax contains error code */
1055 CFI_ADJUST_CFA_OFFSET (14*8)
1056 movq %rsi,13*8(%rsp)
1057 CFI_REL_OFFSET rsi,RSI
1058 movq 14*8(%rsp),%rsi /* load rax from rdi slot */
1059 CFI_REGISTER rax,rsi
1060 movq %rdx,12*8(%rsp)
1061 CFI_REL_OFFSET rdx,RDX
1062 movq %rcx,11*8(%rsp)
1063 CFI_REL_OFFSET rcx,RCX
1064 movq %rsi,10*8(%rsp) /* store rax */
1065 CFI_REL_OFFSET rax,RAX
1067 CFI_REL_OFFSET r8,R8
1069 CFI_REL_OFFSET r9,R9
1071 CFI_REL_OFFSET r10,R10
1073 CFI_REL_OFFSET r11,R11
1075 CFI_REL_OFFSET rbx,RBX
1077 CFI_REL_OFFSET rbp,RBP
1079 CFI_REL_OFFSET r12,R12
1081 CFI_REL_OFFSET r13,R13
1083 CFI_REL_OFFSET r14,R14
1085 CFI_REL_OFFSET r15,R15
1088 je error_kernelspace
1094 CFI_REL_OFFSET rdi,RDI
1096 movq ORIG_RAX(%rsp),%rsi /* get error code */
1097 movq $-1,ORIG_RAX(%rsp)
1099 /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
1103 DISABLE_INTERRUPTS(CLBR_NONE)
1105 GET_THREAD_INFO(%rcx)
1108 LOCKDEP_SYS_EXIT_IRQ
1109 movl TI_flags(%rcx),%edx
1110 movl $_TIF_WORK_MASK,%edi
1118 /* There are two places in the kernel that can potentially fault with
1119 usergs. Handle them here. The exception handlers after
1120 iret run with kernel gs again, so don't set the user space flag.
1121 B stepping K8s sometimes report an truncated RIP for IRET
1122 exceptions returning to compat mode. Check for these here too. */
1123 leaq irq_return(%rip),%rcx
1126 movl %ecx,%ecx /* zero extend */
1129 cmpq $gs_change,RIP(%rsp)
1132 KPROBE_END(error_entry)
1134 /* Reload gs selector with exception handling */
1135 /* edi: new selector */
1136 ENTRY(native_load_gs_index)
1139 CFI_ADJUST_CFA_OFFSET 8
1140 DISABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI))
1144 2: mfence /* workaround */
1147 CFI_ADJUST_CFA_OFFSET -8
1150 ENDPROC(native_load_gs_index)
1152 .section __ex_table,"a"
1154 .quad gs_change,bad_gs
1156 .section .fixup,"ax"
1157 /* running with kernelgs */
1159 SWAPGS /* switch back to user gs */
1166 * Create a kernel thread.
1168 * C extern interface:
1169 * extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
1171 * asm input arguments:
1172 * rdi: fn, rsi: arg, rdx: flags
1174 ENTRY(kernel_thread)
1176 FAKE_STACK_FRAME $child_rip
1179 # rdi: flags, rsi: usp, rdx: will be &pt_regs
1181 orq kernel_thread_flags(%rip),%rdi
1194 * It isn't worth to check for reschedule here,
1195 * so internally to the x86_64 port you can rely on kernel_thread()
1196 * not to reschedule the child before returning, this avoids the need
1197 * of hacks for example to fork off the per-CPU idle tasks.
1198 * [Hopefully no generic code relies on the reschedule -AK]
1204 ENDPROC(kernel_thread)
1207 pushq $0 # fake return address
1210 * Here we are in the child and the registers are set as they were
1211 * at kernel_thread() invocation in the parent.
1223 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
1225 * C extern interface:
1226 * extern long execve(char *name, char **argv, char **envp)
1228 * asm input arguments:
1229 * rdi: name, rsi: argv, rdx: envp
1231 * We want to fallback into:
1232 * extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs *regs)
1234 * do_sys_execve asm fallback arguments:
1235 * rdi: name, rsi: argv, rdx: envp, rcx: fake frame on the stack
1237 ENTRY(kernel_execve)
1243 movq %rax, RAX(%rsp)
1246 je int_ret_from_sys_call
1251 ENDPROC(kernel_execve)
1253 KPROBE_ENTRY(page_fault)
1254 errorentry do_page_fault
1255 KPROBE_END(page_fault)
1257 ENTRY(coprocessor_error)
1258 zeroentry do_coprocessor_error
1259 END(coprocessor_error)
1261 ENTRY(simd_coprocessor_error)
1262 zeroentry do_simd_coprocessor_error
1263 END(simd_coprocessor_error)
1265 ENTRY(device_not_available)
1266 zeroentry do_device_not_available
1267 END(device_not_available)
1269 /* runs on exception stack */
1272 PARAVIRT_ADJUST_EXCEPTION_FRAME
1274 CFI_ADJUST_CFA_OFFSET 8
1275 paranoidentry do_debug, DEBUG_STACK
1279 /* runs on exception stack */
1282 PARAVIRT_ADJUST_EXCEPTION_FRAME
1284 CFI_ADJUST_CFA_OFFSET 8
1285 paranoidentry do_nmi, 0, 0
1286 #ifdef CONFIG_TRACE_IRQFLAGS
1296 PARAVIRT_ADJUST_EXCEPTION_FRAME
1298 CFI_ADJUST_CFA_OFFSET 8
1299 paranoidentry do_int3, DEBUG_STACK
1305 zeroentry do_overflow
1313 zeroentry do_invalid_op
1316 ENTRY(coprocessor_segment_overrun)
1317 zeroentry do_coprocessor_segment_overrun
1318 END(coprocessor_segment_overrun)
1320 /* runs on exception stack */
1323 PARAVIRT_ADJUST_EXCEPTION_FRAME
1324 paranoidentry do_double_fault
1330 errorentry do_invalid_TSS
1333 ENTRY(segment_not_present)
1334 errorentry do_segment_not_present
1335 END(segment_not_present)
1337 /* runs on exception stack */
1338 ENTRY(stack_segment)
1340 PARAVIRT_ADJUST_EXCEPTION_FRAME
1341 paranoidentry do_stack_segment
1346 KPROBE_ENTRY(general_protection)
1347 errorentry do_general_protection
1348 KPROBE_END(general_protection)
1350 ENTRY(alignment_check)
1351 errorentry do_alignment_check
1352 END(alignment_check)
1355 zeroentry do_divide_error
1358 ENTRY(spurious_interrupt_bug)
1359 zeroentry do_spurious_interrupt_bug
1360 END(spurious_interrupt_bug)
1362 #ifdef CONFIG_X86_MCE
1363 /* runs on exception stack */
1364 ENTRY(machine_check)
1366 PARAVIRT_ADJUST_EXCEPTION_FRAME
1368 CFI_ADJUST_CFA_OFFSET 8
1369 paranoidentry do_machine_check
1375 /* Call softirq on interrupt stack. Interrupts are off. */
1379 CFI_ADJUST_CFA_OFFSET 8
1380 CFI_REL_OFFSET rbp,0
1382 CFI_DEF_CFA_REGISTER rbp
1383 incl %gs:pda_irqcount
1384 cmove %gs:pda_irqstackptr,%rsp
1385 push %rbp # backlink for old unwinder
1388 CFI_DEF_CFA_REGISTER rsp
1389 CFI_ADJUST_CFA_OFFSET -8
1390 decl %gs:pda_irqcount
1393 ENDPROC(call_softirq)
1395 KPROBE_ENTRY(ignore_sysret)
1400 ENDPROC(ignore_sysret)
1403 ENTRY(xen_hypervisor_callback)
1404 zeroentry xen_do_hypervisor_callback
1405 END(xen_hypervisor_callback)
1408 # A note on the "critical region" in our callback handler.
1409 # We want to avoid stacking callback handlers due to events occurring
1410 # during handling of the last event. To do this, we keep events disabled
1411 # until we've done all processing. HOWEVER, we must enable events before
1412 # popping the stack frame (can't be done atomically) and so it would still
1413 # be possible to get enough handler activations to overflow the stack.
1414 # Although unlikely, bugs of that kind are hard to track down, so we'd
1415 # like to avoid the possibility.
1416 # So, on entry to the handler we detect whether we interrupted an
1417 # existing activation in its critical region -- if so, we pop the current
1418 # activation and restart the handler using the previous one.
1420 ENTRY(xen_do_hypervisor_callback) # do_hypervisor_callback(struct *pt_regs)
1422 /* Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
1423 see the correct pointer to the pt_regs */
1424 movq %rdi, %rsp # we don't return, adjust the stack frame
1427 11: incl %gs:pda_irqcount
1429 CFI_DEF_CFA_REGISTER rbp
1430 cmovzq %gs:pda_irqstackptr,%rsp
1431 pushq %rbp # backlink for old unwinder
1432 call xen_evtchn_do_upcall
1434 CFI_DEF_CFA_REGISTER rsp
1435 decl %gs:pda_irqcount
1438 END(do_hypervisor_callback)
1441 # Hypervisor uses this for application faults while it executes.
1442 # We get here for two reasons:
1443 # 1. Fault while reloading DS, ES, FS or GS
1444 # 2. Fault while executing IRET
1445 # Category 1 we do not need to fix up as Xen has already reloaded all segment
1446 # registers that could be reloaded and zeroed the others.
1447 # Category 2 we fix up by killing the current process. We cannot use the
1448 # normal Linux return path in this case because if we use the IRET hypercall
1449 # to pop the stack frame we end up in an infinite loop of failsafe callbacks.
1450 # We distinguish between categories by comparing each saved segment register
1451 # with its current contents: any discrepancy means we in category 1.
1453 ENTRY(xen_failsafe_callback)
1454 framesz = (RIP-0x30) /* workaround buggy gas */
1456 CFI_REL_OFFSET rcx, 0
1457 CFI_REL_OFFSET r11, 8
1471 /* All segments match their saved values => Category 2 (Bad IRET). */
1477 CFI_ADJUST_CFA_OFFSET -0x30
1479 CFI_ADJUST_CFA_OFFSET 8
1481 CFI_ADJUST_CFA_OFFSET 8
1483 CFI_ADJUST_CFA_OFFSET 8
1484 jmp general_protection
1486 1: /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
1492 CFI_ADJUST_CFA_OFFSET -0x30
1494 CFI_ADJUST_CFA_OFFSET 8
1498 END(xen_failsafe_callback)
1500 #endif /* CONFIG_XEN */