2 Asm versions of Xen pv-ops, suitable for either direct use or inlining.
3 The inline versions are the same as the direct-use versions, with the
4 pre- and post-amble chopped off.
6 This code is encoded for size rather than absolute efficiency,
7 with a view to being able to inline as much as possible.
9 We only bother with direct forms (ie, vcpu in pda) of the operations
10 here; the indirect forms are better handled in C, since they're
11 generally too large to inline anyway.
14 #include <asm/errno.h>
15 #include <asm/percpu.h>
16 #include <asm/processor-flags.h>
17 #include <asm/segment.h>
19 #include <xen/interface/xen.h>
23 ENTRY(xen_adjust_exception_frame)
28 hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32
36 rip <-- standard iret frame
41 r11 }<-- pushed by hypercall page
51 sysexit is not used for 64-bit processes, so it's
52 only ever used to return to 32-bit compat userspace.
64 RELOC(xen_sysexit, 1b+1)
67 /* We're already on the usermode stack at this point, but still
68 with the kernel gs, so we can easily switch back */
69 movq %rsp, PER_CPU_VAR(old_rsp)
70 movq PER_CPU_VAR(kernel_stack),%rsp
73 pushq PER_CPU_VAR(old_rsp)
78 pushq $VGCF_in_syscall
80 ENDPATCH(xen_sysret64)
81 RELOC(xen_sysret64, 1b+1)
84 /* We're already on the usermode stack at this point, but still
85 with the kernel gs, so we can easily switch back */
86 movq %rsp, PER_CPU_VAR(old_rsp)
87 movq PER_CPU_VAR(kernel_stack), %rsp
90 pushq PER_CPU_VAR(old_rsp)
95 pushq $VGCF_in_syscall
97 ENDPATCH(xen_sysret32)
98 RELOC(xen_sysret32, 1b+1)
101 Xen handles syscall callbacks much like ordinary exceptions,
105 - an iret-like stack frame on the stack (including rcx and r11):
114 In all the entrypoints, we undo all that to make it look
115 like a CPU-generated syscall/sysenter and jump to the normal
119 .macro undo_xen_syscall
125 /* Normal 64-bit system call target */
126 ENTRY(xen_syscall_target)
128 jmp system_call_after_swapgs
129 ENDPROC(xen_syscall_target)
131 #ifdef CONFIG_IA32_EMULATION
133 /* 32-bit compat syscall target */
134 ENTRY(xen_syscall32_target)
136 jmp ia32_cstar_target
137 ENDPROC(xen_syscall32_target)
139 /* 32-bit compat sysenter target */
140 ENTRY(xen_sysenter_target)
142 jmp ia32_sysenter_target
143 ENDPROC(xen_sysenter_target)
145 #else /* !CONFIG_IA32_EMULATION */
147 ENTRY(xen_syscall32_target)
148 ENTRY(xen_sysenter_target)
149 lea 16(%rsp), %rsp /* strip %rcx,%r11 */
151 pushq $VGCF_in_syscall
153 ENDPROC(xen_syscall32_target)
154 ENDPROC(xen_sysenter_target)
156 #endif /* CONFIG_IA32_EMULATION */