2 * linux/arch/x86_64/entry.S
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
6 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
10 * entry.S contains the system-call and fault low-level handling routines.
12 * NOTE: This code handles signal-recognition, which happens every time
13 * after an interrupt and after each system call.
15 * Normal syscalls and interrupts don't save a full stack frame, this is
16 * only done for syscall tracing, signals or fork/exec et.al.
18 * A note on terminology:
19 * - top of stack: Architecture defined interrupt frame from SS to RIP
20 * at the top of the kernel process stack.
21 * - partial stack frame: partially saved registers upto R11.
22 * - full stack frame: Like partial stack frame, but all register saved.
25 * - CFI macros are used to generate dwarf2 unwind information for better
26 * backtraces. They don't change any code.
27 * - SAVE_ALL/RESTORE_ALL - Save/restore all registers
28 * - SAVE_ARGS/RESTORE_ARGS - Save/restore registers that C functions modify.
29 * There are unfortunately lots of special cases where some registers
30 * not touched. The macro is a big mess that should be cleaned up.
31 * - SAVE_REST/RESTORE_REST - Handle the registers not saved by SAVE_ARGS.
32 * Gives a full stack frame.
33 * - ENTRY/END Define functions in the symbol table.
34 * - FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK - Fix up the hardware stack
35 * frame that is otherwise undefined after a SYSCALL
36 * - TRACE_IRQ_* - Trace hard interrupt state for lock debugging.
37 * - errorentry/paranoidentry/zeroentry - Define exception entry points.
40 #include <linux/linkage.h>
41 #include <asm/segment.h>
42 #include <asm/cache.h>
43 #include <asm/errno.h>
44 #include <asm/dwarf2.h>
45 #include <asm/calling.h>
46 #include <asm/asm-offsets.h>
48 #include <asm/unistd.h>
49 #include <asm/thread_info.h>
50 #include <asm/hw_irq.h>
52 #include <asm/irqflags.h>
53 #include <asm/paravirt.h>
54 #include <asm/ftrace.h>
56 /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */
57 #include <linux/elf-em.h>
58 #define AUDIT_ARCH_X86_64 (EM_X86_64|__AUDIT_ARCH_64BIT|__AUDIT_ARCH_LE)
59 #define __AUDIT_ARCH_64BIT 0x80000000
60 #define __AUDIT_ARCH_LE 0x40000000
64 #ifdef CONFIG_FUNCTION_TRACER
65 #ifdef CONFIG_DYNAMIC_FTRACE
72 /* taken from glibc */
84 subq $MCOUNT_INSN_SIZE, %rdi
104 #else /* ! CONFIG_DYNAMIC_FTRACE */
106 cmpq $ftrace_stub, ftrace_trace_function
113 /* taken from glibc */
123 movq 0x38(%rsp), %rdi
125 subq $MCOUNT_INSN_SIZE, %rdi
127 call *ftrace_trace_function
140 #endif /* CONFIG_DYNAMIC_FTRACE */
141 #endif /* CONFIG_FUNCTION_TRACER */
143 #ifndef CONFIG_PREEMPT
144 #define retint_kernel retint_restore_args
147 #ifdef CONFIG_PARAVIRT
148 ENTRY(native_usergs_sysret64)
151 #endif /* CONFIG_PARAVIRT */
154 .macro TRACE_IRQS_IRETQ offset=ARGOFFSET
155 #ifdef CONFIG_TRACE_IRQFLAGS
156 bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */
164 * C code is not supposed to know about undefined top of stack. Every time
165 * a C function with an pt_regs argument is called from the SYSCALL based
166 * fast path FIXUP_TOP_OF_STACK is needed.
167 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
171 /* %rsp:at FRAMEEND */
172 .macro FIXUP_TOP_OF_STACK tmp
173 movq %gs:pda_oldrsp,\tmp
175 movq $__USER_DS,SS(%rsp)
176 movq $__USER_CS,CS(%rsp)
178 movq R11(%rsp),\tmp /* get eflags */
179 movq \tmp,EFLAGS(%rsp)
182 .macro RESTORE_TOP_OF_STACK tmp,offset=0
183 movq RSP-\offset(%rsp),\tmp
184 movq \tmp,%gs:pda_oldrsp
185 movq EFLAGS-\offset(%rsp),\tmp
186 movq \tmp,R11-\offset(%rsp)
189 .macro FAKE_STACK_FRAME child_rip
190 /* push in order ss, rsp, eflags, cs, rip */
192 pushq $__KERNEL_DS /* ss */
193 CFI_ADJUST_CFA_OFFSET 8
194 /*CFI_REL_OFFSET ss,0*/
196 CFI_ADJUST_CFA_OFFSET 8
198 pushq $(1<<9) /* eflags - interrupts on */
199 CFI_ADJUST_CFA_OFFSET 8
200 /*CFI_REL_OFFSET rflags,0*/
201 pushq $__KERNEL_CS /* cs */
202 CFI_ADJUST_CFA_OFFSET 8
203 /*CFI_REL_OFFSET cs,0*/
204 pushq \child_rip /* rip */
205 CFI_ADJUST_CFA_OFFSET 8
207 pushq %rax /* orig rax */
208 CFI_ADJUST_CFA_OFFSET 8
211 .macro UNFAKE_STACK_FRAME
213 CFI_ADJUST_CFA_OFFSET -(6*8)
216 .macro CFI_DEFAULT_STACK start=1
222 CFI_DEF_CFA_OFFSET SS+8
224 CFI_REL_OFFSET r15,R15
225 CFI_REL_OFFSET r14,R14
226 CFI_REL_OFFSET r13,R13
227 CFI_REL_OFFSET r12,R12
228 CFI_REL_OFFSET rbp,RBP
229 CFI_REL_OFFSET rbx,RBX
230 CFI_REL_OFFSET r11,R11
231 CFI_REL_OFFSET r10,R10
234 CFI_REL_OFFSET rax,RAX
235 CFI_REL_OFFSET rcx,RCX
236 CFI_REL_OFFSET rdx,RDX
237 CFI_REL_OFFSET rsi,RSI
238 CFI_REL_OFFSET rdi,RDI
239 CFI_REL_OFFSET rip,RIP
240 /*CFI_REL_OFFSET cs,CS*/
241 /*CFI_REL_OFFSET rflags,EFLAGS*/
242 CFI_REL_OFFSET rsp,RSP
243 /*CFI_REL_OFFSET ss,SS*/
246 * A newly forked process directly context switches into this.
251 push kernel_eflags(%rip)
252 CFI_ADJUST_CFA_OFFSET 8
253 popf # reset kernel eflags
254 CFI_ADJUST_CFA_OFFSET -8
256 GET_THREAD_INFO(%rcx)
257 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),TI_flags(%rcx)
261 testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?
262 je int_ret_from_sys_call
263 testl $_TIF_IA32,TI_flags(%rcx)
264 jnz int_ret_from_sys_call
265 RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
266 jmp ret_from_sys_call
269 call syscall_trace_leave
270 GET_THREAD_INFO(%rcx)
276 * System call entry. Upto 6 arguments in registers are supported.
278 * SYSCALL does not save anything on the stack and does not change the
284 * rax system call number
286 * rcx return address for syscall/sysret, C arg3
289 * r10 arg3 (--> moved to rcx for C)
292 * r11 eflags for syscall/sysret, temporary for C
293 * r12-r15,rbp,rbx saved by C code, not touched.
295 * Interrupts are off on entry.
296 * Only called from user space.
298 * XXX if we had a free scratch register we could save the RSP into the stack frame
299 * and report it properly in ps. Unfortunately we haven't.
301 * When user can change the frames always force IRET. That is because
302 * it deals with uncanonical addresses better. SYSRET has trouble
303 * with them due to bugs in both AMD and Intel CPUs.
309 CFI_DEF_CFA rsp,PDA_STACKOFFSET
311 /*CFI_REGISTER rflags,r11*/
314 * A hypervisor implementation might want to use a label
315 * after the swapgs, so that it can do the swapgs
316 * for the guest and jump here on syscall.
318 ENTRY(system_call_after_swapgs)
320 movq %rsp,%gs:pda_oldrsp
321 movq %gs:pda_kernelstack,%rsp
323 * No need to follow this irqs off/on section - it's straight
326 ENABLE_INTERRUPTS(CLBR_NONE)
328 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
329 movq %rcx,RIP-ARGOFFSET(%rsp)
330 CFI_REL_OFFSET rip,RIP-ARGOFFSET
331 GET_THREAD_INFO(%rcx)
332 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%rcx)
334 system_call_fastpath:
335 cmpq $__NR_syscall_max,%rax
338 call *sys_call_table(,%rax,8) # XXX: rip relative
339 movq %rax,RAX-ARGOFFSET(%rsp)
341 * Syscall return path ending with SYSRET (fast path)
342 * Has incomplete stack frame and undefined top of stack.
345 movl $_TIF_ALLWORK_MASK,%edi
349 GET_THREAD_INFO(%rcx)
350 DISABLE_INTERRUPTS(CLBR_NONE)
352 movl TI_flags(%rcx),%edx
357 * sysretq will re-enable interrupts:
360 movq RIP-ARGOFFSET(%rsp),%rcx
362 RESTORE_ARGS 0,-ARG_SKIP,1
363 /*CFI_REGISTER rflags,r11*/
364 movq %gs:pda_oldrsp, %rsp
368 /* Handle reschedules */
369 /* edx: work, edi: workmask */
371 bt $TIF_NEED_RESCHED,%edx
374 ENABLE_INTERRUPTS(CLBR_NONE)
376 CFI_ADJUST_CFA_OFFSET 8
379 CFI_ADJUST_CFA_OFFSET -8
382 /* Handle a signal */
385 ENABLE_INTERRUPTS(CLBR_NONE)
386 #ifdef CONFIG_AUDITSYSCALL
387 bt $TIF_SYSCALL_AUDIT,%edx
390 /* edx: work flags (arg3) */
391 leaq do_notify_resume(%rip),%rax
392 leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
393 xorl %esi,%esi # oldset -> arg2
394 call ptregscall_common
395 movl $_TIF_WORK_MASK,%edi
396 /* Use IRET because user could have changed frame. This
397 works because ptregscall_common has called FIXUP_TOP_OF_STACK. */
398 DISABLE_INTERRUPTS(CLBR_NONE)
403 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
404 jmp ret_from_sys_call
406 #ifdef CONFIG_AUDITSYSCALL
408 * Fast path for syscall audit without full syscall trace.
409 * We just call audit_syscall_entry() directly, and then
410 * jump back to the normal fast path.
413 movq %r10,%r9 /* 6th arg: 4th syscall arg */
414 movq %rdx,%r8 /* 5th arg: 3rd syscall arg */
415 movq %rsi,%rcx /* 4th arg: 2nd syscall arg */
416 movq %rdi,%rdx /* 3rd arg: 1st syscall arg */
417 movq %rax,%rsi /* 2nd arg: syscall number */
418 movl $AUDIT_ARCH_X86_64,%edi /* 1st arg: audit arch */
419 call audit_syscall_entry
420 LOAD_ARGS 0 /* reload call-clobbered registers */
421 jmp system_call_fastpath
424 * Return fast path for syscall audit. Call audit_syscall_exit()
425 * directly and then jump back to the fast path with TIF_SYSCALL_AUDIT
429 movq %rax,%rsi /* second arg, syscall return value */
430 cmpq $0,%rax /* is it < 0? */
431 setl %al /* 1 if so, 0 if not */
432 movzbl %al,%edi /* zero-extend that into %edi */
433 inc %edi /* first arg, 0->1(AUDITSC_SUCCESS), 1->2(AUDITSC_FAILURE) */
434 call audit_syscall_exit
435 movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
437 #endif /* CONFIG_AUDITSYSCALL */
439 /* Do syscall tracing */
441 #ifdef CONFIG_AUDITSYSCALL
442 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%rcx)
446 movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
447 FIXUP_TOP_OF_STACK %rdi
449 call syscall_trace_enter
451 * Reload arg registers from stack in case ptrace changed them.
452 * We don't reload %rax because syscall_trace_enter() returned
453 * the value it wants us to use in the table lookup.
455 LOAD_ARGS ARGOFFSET, 1
457 cmpq $__NR_syscall_max,%rax
458 ja int_ret_from_sys_call /* RAX(%rsp) set to -ENOSYS above */
459 movq %r10,%rcx /* fixup for C */
460 call *sys_call_table(,%rax,8)
461 movq %rax,RAX-ARGOFFSET(%rsp)
462 /* Use IRET because user could have changed frame */
465 * Syscall return path ending with IRET.
466 * Has correct top of stack, but partial stack frame.
468 .globl int_ret_from_sys_call
469 .globl int_with_check
470 int_ret_from_sys_call:
471 DISABLE_INTERRUPTS(CLBR_NONE)
473 testl $3,CS-ARGOFFSET(%rsp)
474 je retint_restore_args
475 movl $_TIF_ALLWORK_MASK,%edi
476 /* edi: mask to check */
479 GET_THREAD_INFO(%rcx)
480 movl TI_flags(%rcx),%edx
483 andl $~TS_COMPAT,TI_status(%rcx)
486 /* Either reschedule or signal or syscall exit tracking needed. */
487 /* First do a reschedule test. */
488 /* edx: work, edi: workmask */
490 bt $TIF_NEED_RESCHED,%edx
493 ENABLE_INTERRUPTS(CLBR_NONE)
495 CFI_ADJUST_CFA_OFFSET 8
498 CFI_ADJUST_CFA_OFFSET -8
499 DISABLE_INTERRUPTS(CLBR_NONE)
503 /* handle signals and tracing -- both require a full stack frame */
506 ENABLE_INTERRUPTS(CLBR_NONE)
508 /* Check for syscall exit trace */
509 testl $_TIF_WORK_SYSCALL_EXIT,%edx
512 CFI_ADJUST_CFA_OFFSET 8
513 leaq 8(%rsp),%rdi # &ptregs -> arg1
514 call syscall_trace_leave
516 CFI_ADJUST_CFA_OFFSET -8
517 andl $~(_TIF_WORK_SYSCALL_EXIT|_TIF_SYSCALL_EMU),%edi
521 testl $_TIF_DO_NOTIFY_MASK,%edx
523 movq %rsp,%rdi # &ptregs -> arg1
524 xorl %esi,%esi # oldset -> arg2
525 call do_notify_resume
526 1: movl $_TIF_WORK_MASK,%edi
529 DISABLE_INTERRUPTS(CLBR_NONE)
536 * Certain special system calls that need to save a complete full stack frame.
539 .macro PTREGSCALL label,func,arg
542 leaq \func(%rip),%rax
543 leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
544 jmp ptregscall_common
550 PTREGSCALL stub_clone, sys_clone, %r8
551 PTREGSCALL stub_fork, sys_fork, %rdi
552 PTREGSCALL stub_vfork, sys_vfork, %rdi
553 PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
554 PTREGSCALL stub_iopl, sys_iopl, %rsi
556 ENTRY(ptregscall_common)
558 CFI_ADJUST_CFA_OFFSET -8
559 CFI_REGISTER rip, r11
562 CFI_REGISTER rip, r15
563 FIXUP_TOP_OF_STACK %r11
565 RESTORE_TOP_OF_STACK %r11
567 CFI_REGISTER rip, r11
570 CFI_ADJUST_CFA_OFFSET 8
571 CFI_REL_OFFSET rip, 0
574 END(ptregscall_common)
579 CFI_ADJUST_CFA_OFFSET -8
580 CFI_REGISTER rip, r11
582 FIXUP_TOP_OF_STACK %r11
585 RESTORE_TOP_OF_STACK %r11
588 jmp int_ret_from_sys_call
593 * sigreturn is special because it needs to restore all registers on return.
594 * This cannot be done with SYSRET, so use the IRET return path instead.
596 ENTRY(stub_rt_sigreturn)
599 CFI_ADJUST_CFA_OFFSET -8
602 FIXUP_TOP_OF_STACK %r11
603 call sys_rt_sigreturn
604 movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
606 jmp int_ret_from_sys_call
608 END(stub_rt_sigreturn)
611 * initial frame state for interrupts and exceptions
616 CFI_DEF_CFA rsp,SS+8-\ref
617 /*CFI_REL_OFFSET ss,SS-\ref*/
618 CFI_REL_OFFSET rsp,RSP-\ref
619 /*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
620 /*CFI_REL_OFFSET cs,CS-\ref*/
621 CFI_REL_OFFSET rip,RIP-\ref
624 /* initial frame state for interrupts (and exceptions without error code) */
625 #define INTR_FRAME _frame RIP
626 /* initial frame state for exceptions with error code (and interrupts with
627 vector already pushed) */
628 #define XCPT_FRAME _frame ORIG_RAX
631 * Build the entry stubs and pointer table with some assembler magic.
632 * We pack 7 stubs into a single 32-byte chunk, which will fit in a
633 * single cache line on all modern x86 implementations.
635 .section .init.rodata,"a"
639 .p2align CONFIG_X86_L1_CACHE_SHIFT
640 ENTRY(irq_entries_start)
642 vector=FIRST_EXTERNAL_VECTOR
643 .rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7
646 .if vector < NR_VECTORS
647 .if vector != FIRST_EXTERNAL_VECTOR
648 CFI_ADJUST_CFA_OFFSET -8
650 1: pushq $(~vector+0x80) /* Note: always in signed byte range */
651 CFI_ADJUST_CFA_OFFSET 8
652 .if ((vector-FIRST_EXTERNAL_VECTOR)%7) != 6
661 2: jmp common_interrupt
664 END(irq_entries_start)
671 * Interrupt entry/exit.
673 * Interrupt entry points save only callee clobbered registers in fast path.
675 * Entry runs with interrupts off.
678 /* 0(%rsp): ~(interrupt number)+0x80 */
679 .macro interrupt func
680 addq $-0x80,(%rsp) /* Adjust vector to [-256,-1] range */
683 leaq -ARGOFFSET(%rsp),%rdi /* arg1 for handler */
686 * Save rbp twice: One is for marking the stack frame, as usual, and the
687 * other, to fill pt_regs properly. This is because bx comes right
688 * before the last saved register in that structure, and not bp. If the
689 * base pointer were in the place bx is today, this would not be needed.
692 CFI_ADJUST_CFA_OFFSET 8
693 CFI_REL_OFFSET rbp, 0
695 CFI_DEF_CFA_REGISTER rbp
699 /* irqcount is used to check if a CPU is already on an interrupt
700 stack or not. While this is essentially redundant with preempt_count
701 it is a little cheaper to use a separate counter in the PDA
702 (short of moving irq_enter into assembly, which would be too
704 1: incl %gs:pda_irqcount
705 cmoveq %gs:pda_irqstackptr,%rsp
706 push %rbp # backlink for old unwinder
708 * We entered an interrupt context - irqs are off:
714 .p2align CONFIG_X86_L1_CACHE_SHIFT
718 /* 0(%rsp): oldrsp-ARGOFFSET */
720 DISABLE_INTERRUPTS(CLBR_NONE)
722 decl %gs:pda_irqcount
724 CFI_DEF_CFA_REGISTER rsp
725 CFI_ADJUST_CFA_OFFSET -8
727 GET_THREAD_INFO(%rcx)
728 testl $3,CS-ARGOFFSET(%rsp)
731 /* Interrupt came from user space */
733 * Has a correct top of stack, but a partial stack frame
734 * %rcx: thread info. Interrupts off.
736 retint_with_reschedule:
737 movl $_TIF_WORK_MASK,%edi
740 movl TI_flags(%rcx),%edx
745 retint_swapgs: /* return to user-space */
747 * The iretq could re-enable interrupts:
749 DISABLE_INTERRUPTS(CLBR_ANY)
754 retint_restore_args: /* return to kernel space */
755 DISABLE_INTERRUPTS(CLBR_ANY)
757 * The iretq could re-enable interrupts:
766 .section __ex_table, "a"
767 .quad irq_return, bad_iret
770 #ifdef CONFIG_PARAVIRT
774 .section __ex_table,"a"
775 .quad native_iret, bad_iret
782 * The iret traps when the %cs or %ss being restored is bogus.
783 * We've lost the original trap vector and error code.
784 * #GPF is the most likely one to get for an invalid selector.
785 * So pretend we completed the iret and took the #GPF in user mode.
787 * We are now running with the kernel GS after exception recovery.
788 * But error_entry expects us to have user GS to match the user %cs,
794 jmp general_protection
798 /* edi: workmask, edx: work */
801 bt $TIF_NEED_RESCHED,%edx
804 ENABLE_INTERRUPTS(CLBR_NONE)
806 CFI_ADJUST_CFA_OFFSET 8
809 CFI_ADJUST_CFA_OFFSET -8
810 GET_THREAD_INFO(%rcx)
811 DISABLE_INTERRUPTS(CLBR_NONE)
816 testl $_TIF_DO_NOTIFY_MASK,%edx
819 ENABLE_INTERRUPTS(CLBR_NONE)
821 movq $-1,ORIG_RAX(%rsp)
822 xorl %esi,%esi # oldset
823 movq %rsp,%rdi # &pt_regs
824 call do_notify_resume
826 DISABLE_INTERRUPTS(CLBR_NONE)
828 GET_THREAD_INFO(%rcx)
829 jmp retint_with_reschedule
831 #ifdef CONFIG_PREEMPT
832 /* Returning to kernel space. Check if we need preemption */
833 /* rcx: threadinfo. interrupts off. */
835 cmpl $0,TI_preempt_count(%rcx)
836 jnz retint_restore_args
837 bt $TIF_NEED_RESCHED,TI_flags(%rcx)
838 jnc retint_restore_args
839 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
840 jnc retint_restore_args
841 call preempt_schedule_irq
846 END(common_interrupt)
851 .macro apicinterrupt num,func
854 CFI_ADJUST_CFA_OFFSET 8
860 ENTRY(thermal_interrupt)
861 apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
862 END(thermal_interrupt)
864 ENTRY(threshold_interrupt)
865 apicinterrupt THRESHOLD_APIC_VECTOR,mce_threshold_interrupt
866 END(threshold_interrupt)
869 ENTRY(reschedule_interrupt)
870 apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
871 END(reschedule_interrupt)
873 .macro INVALIDATE_ENTRY num
874 ENTRY(invalidate_interrupt\num)
875 apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt
876 END(invalidate_interrupt\num)
888 ENTRY(call_function_interrupt)
889 apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
890 END(call_function_interrupt)
891 ENTRY(call_function_single_interrupt)
892 apicinterrupt CALL_FUNCTION_SINGLE_VECTOR,smp_call_function_single_interrupt
893 END(call_function_single_interrupt)
894 ENTRY(irq_move_cleanup_interrupt)
895 apicinterrupt IRQ_MOVE_CLEANUP_VECTOR,smp_irq_move_cleanup_interrupt
896 END(irq_move_cleanup_interrupt)
899 ENTRY(apic_timer_interrupt)
900 apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
901 END(apic_timer_interrupt)
903 ENTRY(uv_bau_message_intr1)
904 apicinterrupt 220,uv_bau_message_interrupt
905 END(uv_bau_message_intr1)
907 ENTRY(error_interrupt)
908 apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
911 ENTRY(spurious_interrupt)
912 apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
913 END(spurious_interrupt)
916 * Exception entry points.
920 PARAVIRT_ADJUST_EXCEPTION_FRAME
921 pushq $0 /* push error code/oldrax */
922 CFI_ADJUST_CFA_OFFSET 8
923 pushq %rax /* push real oldrax to the rdi slot */
924 CFI_ADJUST_CFA_OFFSET 8
931 .macro errorentry sym
933 PARAVIRT_ADJUST_EXCEPTION_FRAME
935 CFI_ADJUST_CFA_OFFSET 8
942 /* error code is on the stack already */
943 /* handle NMI like exceptions that can happen everywhere */
944 .macro paranoidentry sym, ist=0, irqtrace=1
948 movl $MSR_GS_BASE,%ecx
956 movq %gs:pda_data_offset, %rbp
962 movq ORIG_RAX(%rsp),%rsi
963 movq $-1,ORIG_RAX(%rsp)
965 subq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
969 addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
971 DISABLE_INTERRUPTS(CLBR_NONE)
978 * "Paranoid" exit path from exception stack.
979 * Paranoid because this is used by NMIs and cannot take
980 * any kernel state for granted.
981 * We don't do kernel preemption checks here, because only
982 * NMI should be common and it does not enable IRQs and
983 * cannot get reschedule ticks.
985 * "trace" is 0 for the NMI handler only, because irq-tracing
986 * is fundamentally NMI-unsafe. (we cannot change the soft and
987 * hard flags at once, atomically)
989 .macro paranoidexit trace=1
990 /* ebx: no swapgs flag */
992 testl %ebx,%ebx /* swapgs needed? */
993 jnz paranoid_restore\trace
995 jnz paranoid_userspace\trace
996 paranoid_swapgs\trace:
1001 paranoid_restore\trace:
1004 paranoid_userspace\trace:
1005 GET_THREAD_INFO(%rcx)
1006 movl TI_flags(%rcx),%ebx
1007 andl $_TIF_WORK_MASK,%ebx
1008 jz paranoid_swapgs\trace
1009 movq %rsp,%rdi /* &pt_regs */
1011 movq %rax,%rsp /* switch stack for scheduling */
1012 testl $_TIF_NEED_RESCHED,%ebx
1013 jnz paranoid_schedule\trace
1014 movl %ebx,%edx /* arg3: thread flags */
1018 ENABLE_INTERRUPTS(CLBR_NONE)
1019 xorl %esi,%esi /* arg2: oldset */
1020 movq %rsp,%rdi /* arg1: &pt_regs */
1021 call do_notify_resume
1022 DISABLE_INTERRUPTS(CLBR_NONE)
1026 jmp paranoid_userspace\trace
1027 paranoid_schedule\trace:
1031 ENABLE_INTERRUPTS(CLBR_ANY)
1033 DISABLE_INTERRUPTS(CLBR_ANY)
1037 jmp paranoid_userspace\trace
1042 * Exception entry point. This expects an error code/orig_rax on the stack
1043 * and the exception handler in %rax.
1045 KPROBE_ENTRY(error_entry)
1047 CFI_REL_OFFSET rax,0
1048 /* rdi slot contains rax, oldrax contains error code */
1051 CFI_ADJUST_CFA_OFFSET (14*8)
1052 movq %rsi,13*8(%rsp)
1053 CFI_REL_OFFSET rsi,RSI
1054 movq 14*8(%rsp),%rsi /* load rax from rdi slot */
1055 CFI_REGISTER rax,rsi
1056 movq %rdx,12*8(%rsp)
1057 CFI_REL_OFFSET rdx,RDX
1058 movq %rcx,11*8(%rsp)
1059 CFI_REL_OFFSET rcx,RCX
1060 movq %rsi,10*8(%rsp) /* store rax */
1061 CFI_REL_OFFSET rax,RAX
1063 CFI_REL_OFFSET r8,R8
1065 CFI_REL_OFFSET r9,R9
1067 CFI_REL_OFFSET r10,R10
1069 CFI_REL_OFFSET r11,R11
1071 CFI_REL_OFFSET rbx,RBX
1073 CFI_REL_OFFSET rbp,RBP
1075 CFI_REL_OFFSET r12,R12
1077 CFI_REL_OFFSET r13,R13
1079 CFI_REL_OFFSET r14,R14
1081 CFI_REL_OFFSET r15,R15
1084 je error_kernelspace
1090 CFI_REL_OFFSET rdi,RDI
1092 movq ORIG_RAX(%rsp),%rsi /* get error code */
1093 movq $-1,ORIG_RAX(%rsp)
1095 /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
1099 DISABLE_INTERRUPTS(CLBR_NONE)
1101 GET_THREAD_INFO(%rcx)
1104 LOCKDEP_SYS_EXIT_IRQ
1105 movl TI_flags(%rcx),%edx
1106 movl $_TIF_WORK_MASK,%edi
1114 /* There are two places in the kernel that can potentially fault with
1115 usergs. Handle them here. The exception handlers after
1116 iret run with kernel gs again, so don't set the user space flag.
1117 B stepping K8s sometimes report an truncated RIP for IRET
1118 exceptions returning to compat mode. Check for these here too. */
1119 leaq irq_return(%rip),%rcx
1122 movl %ecx,%ecx /* zero extend */
1125 cmpq $gs_change,RIP(%rsp)
1128 KPROBE_END(error_entry)
1130 /* Reload gs selector with exception handling */
1131 /* edi: new selector */
1132 ENTRY(native_load_gs_index)
1135 CFI_ADJUST_CFA_OFFSET 8
1136 DISABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI))
1140 2: mfence /* workaround */
1143 CFI_ADJUST_CFA_OFFSET -8
1146 ENDPROC(native_load_gs_index)
1148 .section __ex_table,"a"
1150 .quad gs_change,bad_gs
1152 .section .fixup,"ax"
1153 /* running with kernelgs */
1155 SWAPGS /* switch back to user gs */
1162 * Create a kernel thread.
1164 * C extern interface:
1165 * extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
1167 * asm input arguments:
1168 * rdi: fn, rsi: arg, rdx: flags
1170 ENTRY(kernel_thread)
1172 FAKE_STACK_FRAME $child_rip
1175 # rdi: flags, rsi: usp, rdx: will be &pt_regs
1177 orq kernel_thread_flags(%rip),%rdi
1190 * It isn't worth to check for reschedule here,
1191 * so internally to the x86_64 port you can rely on kernel_thread()
1192 * not to reschedule the child before returning, this avoids the need
1193 * of hacks for example to fork off the per-CPU idle tasks.
1194 * [Hopefully no generic code relies on the reschedule -AK]
1200 ENDPROC(kernel_thread)
1203 pushq $0 # fake return address
1206 * Here we are in the child and the registers are set as they were
1207 * at kernel_thread() invocation in the parent.
1219 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
1221 * C extern interface:
1222 * extern long execve(char *name, char **argv, char **envp)
1224 * asm input arguments:
1225 * rdi: name, rsi: argv, rdx: envp
1227 * We want to fallback into:
1228 * extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs *regs)
1230 * do_sys_execve asm fallback arguments:
1231 * rdi: name, rsi: argv, rdx: envp, rcx: fake frame on the stack
1233 ENTRY(kernel_execve)
1239 movq %rax, RAX(%rsp)
1242 je int_ret_from_sys_call
1247 ENDPROC(kernel_execve)
1249 KPROBE_ENTRY(page_fault)
1250 errorentry do_page_fault
1251 KPROBE_END(page_fault)
1253 ENTRY(coprocessor_error)
1254 zeroentry do_coprocessor_error
1255 END(coprocessor_error)
1257 ENTRY(simd_coprocessor_error)
1258 zeroentry do_simd_coprocessor_error
1259 END(simd_coprocessor_error)
1261 ENTRY(device_not_available)
1262 zeroentry do_device_not_available
1263 END(device_not_available)
1265 /* runs on exception stack */
1268 PARAVIRT_ADJUST_EXCEPTION_FRAME
1270 CFI_ADJUST_CFA_OFFSET 8
1271 paranoidentry do_debug, DEBUG_STACK
1275 /* runs on exception stack */
1278 PARAVIRT_ADJUST_EXCEPTION_FRAME
1280 CFI_ADJUST_CFA_OFFSET 8
1281 paranoidentry do_nmi, 0, 0
1282 #ifdef CONFIG_TRACE_IRQFLAGS
1292 PARAVIRT_ADJUST_EXCEPTION_FRAME
1294 CFI_ADJUST_CFA_OFFSET 8
1295 paranoidentry do_int3, DEBUG_STACK
1301 zeroentry do_overflow
1309 zeroentry do_invalid_op
1312 ENTRY(coprocessor_segment_overrun)
1313 zeroentry do_coprocessor_segment_overrun
1314 END(coprocessor_segment_overrun)
1316 /* runs on exception stack */
1319 PARAVIRT_ADJUST_EXCEPTION_FRAME
1320 paranoidentry do_double_fault
1326 errorentry do_invalid_TSS
1329 ENTRY(segment_not_present)
1330 errorentry do_segment_not_present
1331 END(segment_not_present)
1333 /* runs on exception stack */
1334 ENTRY(stack_segment)
1336 PARAVIRT_ADJUST_EXCEPTION_FRAME
1337 paranoidentry do_stack_segment
1342 KPROBE_ENTRY(general_protection)
1343 errorentry do_general_protection
1344 KPROBE_END(general_protection)
1346 ENTRY(alignment_check)
1347 errorentry do_alignment_check
1348 END(alignment_check)
1351 zeroentry do_divide_error
1354 ENTRY(spurious_interrupt_bug)
1355 zeroentry do_spurious_interrupt_bug
1356 END(spurious_interrupt_bug)
1358 #ifdef CONFIG_X86_MCE
1359 /* runs on exception stack */
1360 ENTRY(machine_check)
1362 PARAVIRT_ADJUST_EXCEPTION_FRAME
1364 CFI_ADJUST_CFA_OFFSET 8
1365 paranoidentry do_machine_check
1371 /* Call softirq on interrupt stack. Interrupts are off. */
1375 CFI_ADJUST_CFA_OFFSET 8
1376 CFI_REL_OFFSET rbp,0
1378 CFI_DEF_CFA_REGISTER rbp
1379 incl %gs:pda_irqcount
1380 cmove %gs:pda_irqstackptr,%rsp
1381 push %rbp # backlink for old unwinder
1384 CFI_DEF_CFA_REGISTER rsp
1385 CFI_ADJUST_CFA_OFFSET -8
1386 decl %gs:pda_irqcount
1389 ENDPROC(call_softirq)
1391 KPROBE_ENTRY(ignore_sysret)
1396 ENDPROC(ignore_sysret)
1399 ENTRY(xen_hypervisor_callback)
1400 zeroentry xen_do_hypervisor_callback
1401 END(xen_hypervisor_callback)
1404 # A note on the "critical region" in our callback handler.
1405 # We want to avoid stacking callback handlers due to events occurring
1406 # during handling of the last event. To do this, we keep events disabled
1407 # until we've done all processing. HOWEVER, we must enable events before
1408 # popping the stack frame (can't be done atomically) and so it would still
1409 # be possible to get enough handler activations to overflow the stack.
1410 # Although unlikely, bugs of that kind are hard to track down, so we'd
1411 # like to avoid the possibility.
1412 # So, on entry to the handler we detect whether we interrupted an
1413 # existing activation in its critical region -- if so, we pop the current
1414 # activation and restart the handler using the previous one.
1416 ENTRY(xen_do_hypervisor_callback) # do_hypervisor_callback(struct *pt_regs)
1418 /* Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
1419 see the correct pointer to the pt_regs */
1420 movq %rdi, %rsp # we don't return, adjust the stack frame
1423 11: incl %gs:pda_irqcount
1425 CFI_DEF_CFA_REGISTER rbp
1426 cmovzq %gs:pda_irqstackptr,%rsp
1427 pushq %rbp # backlink for old unwinder
1428 call xen_evtchn_do_upcall
1430 CFI_DEF_CFA_REGISTER rsp
1431 decl %gs:pda_irqcount
1434 END(do_hypervisor_callback)
1437 # Hypervisor uses this for application faults while it executes.
1438 # We get here for two reasons:
1439 # 1. Fault while reloading DS, ES, FS or GS
1440 # 2. Fault while executing IRET
1441 # Category 1 we do not need to fix up as Xen has already reloaded all segment
1442 # registers that could be reloaded and zeroed the others.
1443 # Category 2 we fix up by killing the current process. We cannot use the
1444 # normal Linux return path in this case because if we use the IRET hypercall
1445 # to pop the stack frame we end up in an infinite loop of failsafe callbacks.
1446 # We distinguish between categories by comparing each saved segment register
1447 # with its current contents: any discrepancy means we in category 1.
1449 ENTRY(xen_failsafe_callback)
1450 framesz = (RIP-0x30) /* workaround buggy gas */
1452 CFI_REL_OFFSET rcx, 0
1453 CFI_REL_OFFSET r11, 8
1467 /* All segments match their saved values => Category 2 (Bad IRET). */
1473 CFI_ADJUST_CFA_OFFSET -0x30
1475 CFI_ADJUST_CFA_OFFSET 8
1477 CFI_ADJUST_CFA_OFFSET 8
1479 CFI_ADJUST_CFA_OFFSET 8
1480 jmp general_protection
1482 1: /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
1488 CFI_ADJUST_CFA_OFFSET -0x30
1490 CFI_ADJUST_CFA_OFFSET 8
1494 END(xen_failsafe_callback)
1496 #endif /* CONFIG_XEN */