entry_32.S revision 34273f41d57ee8d854dcd2a1d754cbb546cb548f
11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (C) 1991, 1992 Linus Torvalds 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * entry.S contains the system-call and fault low-level handling routines. 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This also contains the timer-interrupt handler, as well as all interrupts 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * and faults that can result in a task-switch. 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * NOTE: This code handles signal-recognition, which happens every time 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * after a timer-interrupt and after each system call. 131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * I changed all the .align's to 4 (16 byte alignment), as that's faster 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * on a 486. 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 17889f21ce272e38db19c8114a7e0a5793d4590077Andi Kleen * Stack layout in 'syscall_exit': 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * ptrace needs to have all regs on the stack. 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * if the order here is changed, it needs to be 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * updated in fork.c:copy_process, signal.c:do_signal, 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * ptrace.c and ptrace.h 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 0(%esp) - %ebx 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 4(%esp) - %ecx 251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 8(%esp) - %edx 261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * C(%esp) - %esi 271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 10(%esp) - %edi 281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 14(%esp) - %ebp 291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 18(%esp) - %eax 301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1C(%esp) - %ds 311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 20(%esp) - %es 32464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge * 24(%esp) - %fs 33ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS 34ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * 2C(%esp) - orig_eax 35ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * 30(%esp) - %eip 36ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * 34(%esp) - %cs 37ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * 38(%esp) - %eflags 38ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * 3C(%esp) - %oldesp 39ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * 40(%esp) - %oldss 401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * "current" is in register %ebx during any slow entries. 421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/linkage.h> 45d7e7528bcd456f5c36ad4a202ccfb43c5aa98bc4Eric Paris#include <linux/err.h> 461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/thread_info.h> 4755f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar#include <asm/irqflags.h> 481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/errno.h> 491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/segment.h> 501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/smp.h> 510341c14da49e7b93d2998926f6ac89a3129e3fa1Jeremy Fitzhardinge#include <asm/page_types.h> 52be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev#include <asm/percpu.h> 53fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich#include <asm/dwarf2.h> 54ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov#include <asm/processor-flags.h> 55395a59d0f8e86bb39cd700c3d185d30c670bb958Abhishek Sagar#include <asm/ftrace.h> 569b7dc567d03d74a1fbae84e88949b6a60d922d82Thomas Gleixner#include <asm/irq_vectors.h> 5740d2e76315da38993129090dc5d56377e573c312Brian Gerst#include <asm/cpufeature.h> 58b4ca46e4e82a0a5976fe5eab85be585d75f8202fAndy Lutomirski#include <asm/alternative-asm.h> 596837a54dd6127f055dcb26d00fee0df05c07a674H. Peter Anvin#include <asm/asm.h> 60e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin#include <asm/smap.h> 611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 62af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */ 63af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#include <linux/elf-em.h> 64af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#define AUDIT_ARCH_I386 (EM_386|__AUDIT_ARCH_LE) 65af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#define __AUDIT_ARCH_LE 0x40000000 66af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath 67af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#ifndef CONFIG_AUDITSYSCALL 68af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#define sysenter_audit syscall_trace_entry 69af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#define sysexit_audit syscall_exit_work 70af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#endif 71af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath 72ea7145477a461e09d8d194cac4b996dc4f449107Jiri Olsa .section .entry.text, "ax" 73ea7145477a461e09d8d194cac4b996dc4f449107Jiri Olsa 74139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell/* 75139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * We use macros for low-level operations which need to be overridden 76139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * for paravirtualization. The following will never clobber any registers: 77139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * INTERRUPT_RETURN (aka. "iret") 78139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") 79d75cd22fdd5f7d203fb60014d426942df33dd9a6Jeremy Fitzhardinge * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit"). 80139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * 81139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must 82139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). 83139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * Allowing a register to be clobbered can shrink the paravirt replacement 84139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * enough to patch inline, increasing performance. 85139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell */ 86139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell 871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef CONFIG_PREEMPT 88139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell#define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF 891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#else 90139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell#define preempt_stop(clobbers) 912e04bc76560decd9270be2a805927316f617ef56Alexander van Heukelum#define resume_kernel restore_all 921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9455f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar.macro TRACE_IRQS_IRET 9555f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar#ifdef CONFIG_TRACE_IRQFLAGS 96ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off? 9755f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar jz 1f 9855f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_ON 9955f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar1: 10055f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar#endif 10155f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar.endm 10255f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar 103ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo/* 104ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * User gs save/restore 105ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * 106ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * %gs is used for userland TLS and kernel only uses it for stack 107ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * canary which is required to be at %gs:20 by gcc. Read the comment 108ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * at the top of stackprotector.h for more info. 109ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * 110ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * Local labels 98 and 99 are used. 111ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo */ 112ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo#ifdef CONFIG_X86_32_LAZY_GS 113ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo 114ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo /* unfortunately push/pop can't be no-op */ 115ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro PUSH_GS 116df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $0 117ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 118ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro POP_GS pop=0 119ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo addl $(4 + \pop), %esp 120ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo CFI_ADJUST_CFA_OFFSET -(4 + \pop) 121ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 122ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro POP_GS_EX 123ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 124ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo 125ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo /* all the rest are no-op */ 126ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro PTGS_TO_GS 127ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 128ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro PTGS_TO_GS_EX 129ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 130ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro GS_TO_REG reg 131ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 132ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro REG_TO_PTGS reg 133ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 134ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro SET_KERNEL_GS reg 135ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 136ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo 137ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo#else /* CONFIG_X86_32_LAZY_GS */ 138ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo 139ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro PUSH_GS 140df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %gs 141ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo /*CFI_REL_OFFSET gs, 0*/ 142ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 143ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo 144ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro POP_GS pop=0 145df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich98: popl_cfi %gs 146ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo /*CFI_RESTORE gs*/ 147ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo .if \pop <> 0 148ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo add $\pop, %esp 149ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo CFI_ADJUST_CFA_OFFSET -\pop 150ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo .endif 151ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 152ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro POP_GS_EX 153ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.pushsection .fixup, "ax" 154ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo99: movl $0, (%esp) 155ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo jmp 98b 156ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.popsection 1576837a54dd6127f055dcb26d00fee0df05c07a674H. Peter Anvin _ASM_EXTABLE(98b,99b) 158ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 159ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo 160ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro PTGS_TO_GS 161ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo98: mov PT_GS(%esp), %gs 162ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 163ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro PTGS_TO_GS_EX 164ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.pushsection .fixup, "ax" 165ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo99: movl $0, PT_GS(%esp) 166ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo jmp 98b 167ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.popsection 1686837a54dd6127f055dcb26d00fee0df05c07a674H. Peter Anvin _ASM_EXTABLE(98b,99b) 169ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 170ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo 171ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro GS_TO_REG reg 172ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo movl %gs, \reg 173ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo /*CFI_REGISTER gs, \reg*/ 174ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 175ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro REG_TO_PTGS reg 176ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo movl \reg, PT_GS(%esp) 177ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo /*CFI_REL_OFFSET gs, PT_GS*/ 178ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 179ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro SET_KERNEL_GS reg 18060a5317ff0f42dd313094b88f809f63041568b08Tejun Heo movl $(__KERNEL_STACK_CANARY), \reg 181ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo movl \reg, %gs 182ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 183ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo 184ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo#endif /* CONFIG_X86_32_LAZY_GS */ 185ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo 186f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro SAVE_ALL 187f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo cld 188ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo PUSH_GS 189df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %fs 190f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_REL_OFFSET fs, 0;*/ 191df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %es 192f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_REL_OFFSET es, 0;*/ 193df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %ds 194f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_REL_OFFSET ds, 0;*/ 195df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %eax 196f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_REL_OFFSET eax, 0 197df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %ebp 198f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_REL_OFFSET ebp, 0 199df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %edi 200f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_REL_OFFSET edi, 0 201df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %esi 202f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_REL_OFFSET esi, 0 203df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %edx 204f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_REL_OFFSET edx, 0 205df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %ecx 206f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_REL_OFFSET ecx, 0 207df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %ebx 208f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_REL_OFFSET ebx, 0 209f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl $(__USER_DS), %edx 210f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl %edx, %ds 211f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl %edx, %es 212f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl $(__KERNEL_PERCPU), %edx 213464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge movl %edx, %fs 214ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo SET_KERNEL_GS %edx 215f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 2161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 217f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro RESTORE_INT_REGS 218df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich popl_cfi %ebx 219f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_RESTORE ebx 220df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich popl_cfi %ecx 221f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_RESTORE ecx 222df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich popl_cfi %edx 223f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_RESTORE edx 224df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich popl_cfi %esi 225f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_RESTORE esi 226df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich popl_cfi %edi 227f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_RESTORE edi 228df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich popl_cfi %ebp 229f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_RESTORE ebp 230df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich popl_cfi %eax 231fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_RESTORE eax 232f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 2331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 234ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro RESTORE_REGS pop=0 235f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo RESTORE_INT_REGS 236df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich1: popl_cfi %ds 237f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_RESTORE ds;*/ 238df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich2: popl_cfi %es 239f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_RESTORE es;*/ 240df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich3: popl_cfi %fs 241f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_RESTORE fs;*/ 242ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo POP_GS \pop 243f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.pushsection .fixup, "ax" 244f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo4: movl $0, (%esp) 245f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo jmp 1b 246f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo5: movl $0, (%esp) 247f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo jmp 2b 248f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo6: movl $0, (%esp) 249f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo jmp 3b 250f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge.popsection 2516837a54dd6127f055dcb26d00fee0df05c07a674H. Peter Anvin _ASM_EXTABLE(1b,4b) 2526837a54dd6127f055dcb26d00fee0df05c07a674H. Peter Anvin _ASM_EXTABLE(2b,5b) 2536837a54dd6127f055dcb26d00fee0df05c07a674H. Peter Anvin _ASM_EXTABLE(3b,6b) 254ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo POP_GS_EX 255f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 2561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 257f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro RING0_INT_FRAME 258f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_STARTPROC simple 259f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_SIGNAL_FRAME 260f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_DEF_CFA esp, 3*4 261f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_OFFSET cs, -2*4;*/ 262fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_OFFSET eip, -3*4 263f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 264fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich 265f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro RING0_EC_FRAME 266f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_STARTPROC simple 267f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_SIGNAL_FRAME 268f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_DEF_CFA esp, 4*4 269f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_OFFSET cs, -2*4;*/ 270fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_OFFSET eip, -3*4 271f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 272fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich 273f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro RING0_PTREGS_FRAME 274f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_STARTPROC simple 275f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_SIGNAL_FRAME 276f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_DEF_CFA esp, PT_OLDESP-PT_EBX 277f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/ 278f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_OFFSET eip, PT_EIP-PT_OLDESP 279f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/ 280f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/ 281f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_OFFSET eax, PT_EAX-PT_OLDESP 282f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_OFFSET ebp, PT_EBP-PT_OLDESP 283f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_OFFSET edi, PT_EDI-PT_OLDESP 284f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_OFFSET esi, PT_ESI-PT_OLDESP 285f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_OFFSET edx, PT_EDX-PT_OLDESP 286f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_OFFSET ecx, PT_ECX-PT_OLDESP 287eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge CFI_OFFSET ebx, PT_EBX-PT_OLDESP 288f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 2891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(ret_from_fork) 291fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_STARTPROC 292df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %eax 2931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call schedule_tail 2941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 295df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich popl_cfi %eax 296df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $0x0202 # Reset kernel eflags 297df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich popfl_cfi 2981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp syscall_exit 299fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 30047a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(ret_from_fork) 3011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 30222e2430d60dbdfcdd732a086e9ef2dbd74c266d1Al ViroENTRY(ret_from_kernel_thread) 30322e2430d60dbdfcdd732a086e9ef2dbd74c266d1Al Viro CFI_STARTPROC 30422e2430d60dbdfcdd732a086e9ef2dbd74c266d1Al Viro pushl_cfi %eax 30522e2430d60dbdfcdd732a086e9ef2dbd74c266d1Al Viro call schedule_tail 3066783eaa2e1253fbcbe2c2f6bb4c843abf1343cafAl Viro GET_THREAD_INFO(%ebp) 30722e2430d60dbdfcdd732a086e9ef2dbd74c266d1Al Viro popl_cfi %eax 30822e2430d60dbdfcdd732a086e9ef2dbd74c266d1Al Viro pushl_cfi $0x0202 # Reset kernel eflags 30922e2430d60dbdfcdd732a086e9ef2dbd74c266d1Al Viro popfl_cfi 31022e2430d60dbdfcdd732a086e9ef2dbd74c266d1Al Viro movl PT_EBP(%esp),%eax 31122e2430d60dbdfcdd732a086e9ef2dbd74c266d1Al Viro call *PT_EBX(%esp) 31222e2430d60dbdfcdd732a086e9ef2dbd74c266d1Al Viro movl $0,PT_EAX(%esp) 3136783eaa2e1253fbcbe2c2f6bb4c843abf1343cafAl Viro jmp syscall_exit 31422e2430d60dbdfcdd732a086e9ef2dbd74c266d1Al Viro CFI_ENDPROC 31522e2430d60dbdfcdd732a086e9ef2dbd74c266d1Al ViroENDPROC(ret_from_kernel_thread) 3166783eaa2e1253fbcbe2c2f6bb4c843abf1343cafAl Viro 3171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 318a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu * Interrupt exit functions should be protected against kprobes 319a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu */ 320a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu .pushsection .kprobes.text, "ax" 321a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu/* 3221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Return to user mode is not as complex as all this looks, 3231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * but we want the default path for a system call return to 3241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * go as quickly as possible which is why some of this is 3251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * less clear than it otherwise should be. 3261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 3271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # userspace resumption stub bypassing syscall exit tracing 3291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 330fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_PTREGS_FRAME 3311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsret_from_exception: 332139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell preempt_stop(CLBR_ANY) 3331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsret_from_intr: 3341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 33529a2e2836ff9ea65a603c89df217f4198973a74fDmitry Adamushko#ifdef CONFIG_VM86 336eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS 337eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movb PT_CS(%esp), %al 338ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax 33929a2e2836ff9ea65a603c89df217f4198973a74fDmitry Adamushko#else 34029a2e2836ff9ea65a603c89df217f4198973a74fDmitry Adamushko /* 3416783eaa2e1253fbcbe2c2f6bb4c843abf1343cafAl Viro * We can be coming here from child spawned by kernel_thread(). 34229a2e2836ff9ea65a603c89df217f4198973a74fDmitry Adamushko */ 34329a2e2836ff9ea65a603c89df217f4198973a74fDmitry Adamushko movl PT_CS(%esp), %eax 34429a2e2836ff9ea65a603c89df217f4198973a74fDmitry Adamushko andl $SEGMENT_RPL_MASK, %eax 34529a2e2836ff9ea65a603c89df217f4198973a74fDmitry Adamushko#endif 34678be3706b21a232310590fe00258b224177ac05fRusty Russell cmpl $USER_RPL, %eax 34778be3706b21a232310590fe00258b224177ac05fRusty Russell jb resume_kernel # not returning to v8086 or userspace 348f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge 3491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(resume_userspace) 350c7e872e7da5514d014707a407ea562d197cc0136Peter Zijlstra LOCKDEP_SYS_EXIT 351139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 3521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # setting need_resched or sigpending 3531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # between sampling and the iret 354e32e58a96de4ac35a03349db2ab69f263ded958fPeter Zijlstra TRACE_IRQS_OFF 3551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx 3561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds andl $_TIF_WORK_MASK, %ecx # is there any work to be done on 3571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # int/exception return? 3581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne work_pending 3591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp restore_all 36047a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(ret_from_exception) 3611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef CONFIG_PREEMPT 3631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(resume_kernel) 364139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_ANY) 3651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsneed_resched: 366c2daa3bed53a81171cf8c1a36db798e82b91afe8Peter Zijlstra cmpl $0,PER_CPU_VAR(__preempt_count) 367c2daa3bed53a81171cf8c1a36db798e82b91afe8Peter Zijlstra jnz restore_all 368ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off (exception path) ? 3691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz restore_all 3701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call preempt_schedule_irq 3711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp need_resched 37247a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(resume_kernel) 3731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 374fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 375a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu/* 376a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu * End of kprobes section 377a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu */ 378a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu .popsection 3791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* SYSENTER_RETURN points to after the "sysenter" instruction in 3811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */ 3821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # sysenter call handler stub 3840aa97fb22624f18e5925d702ab0364d3838cfd91Roland McGrathENTRY(ia32_sysenter_target) 385fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_STARTPROC simple 386adf1423698f00d00b267f7dca8231340ce7d65efJan Beulich CFI_SIGNAL_FRAME 387fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_DEF_CFA esp, 0 388fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REGISTER esp, ebp 389faca62273b602ab482fb7d3d940dbf41ef08b00eH. Peter Anvin movl TSS_sysenter_sp0(%esp),%esp 3901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssysenter_past_esp: 39155f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar /* 392d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge * Interrupts are disabled here, but we can't trace it until 393d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge * enough kernel state to call TRACE_IRQS_OFF can be called - but 394d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge * we immediately enable interrupts at that point anyway. 39555f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar */ 3963234282f33b29d349bcada40204fc7c8fda7fe72Jan Beulich pushl_cfi $__USER_DS 397fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_REL_OFFSET ss, 0*/ 398df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %ebp 399fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET esp, 0 400df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushfl_cfi 401d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge orl $X86_EFLAGS_IF, (%esp) 4023234282f33b29d349bcada40204fc7c8fda7fe72Jan Beulich pushl_cfi $__USER_CS 403fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_REL_OFFSET cs, 0*/ 404e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar /* 405e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar * Push current_thread_info()->sysenter_return to the stack. 406e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar * A tiny bit of offset fixup is necessary - 4*4 means the 4 words 407e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar * pushed above; +8 corresponds to copy_thread's esp0 setting. 408e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar */ 4097bf04be8f48ceeeffa5b5a79734d6d6e0d59e5f8Stratos Psomadakis pushl_cfi ((TI_sysenter_return)-THREAD_SIZE+8+4*4)(%esp) 410fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET eip, 0 4111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 412df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %eax 413d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge SAVE_ALL 414d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge ENABLE_INTERRUPTS(CLBR_NONE) 415d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge 4161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 4171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Load the potential sixth argument from user stack. 4181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Careful about security. 4191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 4201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $__PAGE_OFFSET-3,%ebp 4211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jae syscall_fault 422e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_STAC 4231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds1: movl (%ebp),%ebp 424e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 425d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge movl %ebp,PT_EBP(%esp) 4266837a54dd6127f055dcb26d00fee0df05c07a674H. Peter Anvin _ASM_EXTABLE(1b,syscall_fault) 4271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 4291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 43088200bc28da38bcda1cb1bd218216e83b426d8a8Jaswinder Singh Rajput testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp) 431af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jnz sysenter_audit 432af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrathsysenter_do_call: 433303395ac3bf3e2cb488435537d416bc840438fcbH. Peter Anvin cmpl $(NR_syscalls), %eax 4341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jae syscall_badsys 4351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call *sys_call_table(,%eax,4) 436eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl %eax,PT_EAX(%esp) 437c7e872e7da5514d014707a407ea562d197cc0136Peter Zijlstra LOCKDEP_SYS_EXIT 43842c24fa22e86365055fc931d833f26165e687c19Jeremy Fitzhardinge DISABLE_INTERRUPTS(CLBR_ANY) 43955f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 4401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx 44188200bc28da38bcda1cb1bd218216e83b426d8a8Jaswinder Singh Rajput testl $_TIF_ALLWORK_MASK, %ecx 442af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jne sysexit_audit 443af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrathsysenter_exit: 4441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* if something modifies registers it must also disable sysexit */ 445eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_EIP(%esp), %edx 446eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_OLDESP(%esp), %ecx 4471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %ebp,%ebp 44855f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_ON 449464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge1: mov PT_FS(%esp), %fs 450ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo PTGS_TO_GS 451d75cd22fdd5f7d203fb60014d426942df33dd9a6Jeremy Fitzhardinge ENABLE_INTERRUPTS_SYSEXIT 452af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath 453af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#ifdef CONFIG_AUDITSYSCALL 454af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrathsysenter_audit: 45588200bc28da38bcda1cb1bd218216e83b426d8a8Jaswinder Singh Rajput testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%ebp) 456af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jnz syscall_trace_entry 457af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath addl $4,%esp 458af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath CFI_ADJUST_CFA_OFFSET -4 459af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath /* %esi already in 8(%esp) 6th arg: 4th syscall arg */ 460af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath /* %edx already in 4(%esp) 5th arg: 3rd syscall arg */ 461af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath /* %ecx already in 0(%esp) 4th arg: 2nd syscall arg */ 462af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl %ebx,%ecx /* 3rd arg: 1st syscall arg */ 463af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl %eax,%edx /* 2nd arg: syscall number */ 464af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl $AUDIT_ARCH_I386,%eax /* 1st arg: audit arch */ 465b05d8447e7821695bc2fa3359431f7a664232743Eric Paris call __audit_syscall_entry 466df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %ebx 467af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl PT_EAX(%esp),%eax /* reload syscall number */ 468af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jmp sysenter_do_call 469af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath 470af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrathsysexit_audit: 47188200bc28da38bcda1cb1bd218216e83b426d8a8Jaswinder Singh Rajput testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx 472af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jne syscall_exit_work 473af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath TRACE_IRQS_ON 474af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath ENABLE_INTERRUPTS(CLBR_ANY) 475af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl %eax,%edx /* second arg, syscall return value */ 476d7e7528bcd456f5c36ad4a202ccfb43c5aa98bc4Eric Paris cmpl $-MAX_ERRNO,%eax /* is it an error ? */ 477d7e7528bcd456f5c36ad4a202ccfb43c5aa98bc4Eric Paris setbe %al /* 1 if so, 0 if not */ 478af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movzbl %al,%eax /* zero-extend that */ 479d7e7528bcd456f5c36ad4a202ccfb43c5aa98bc4Eric Paris call __audit_syscall_exit 480af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath DISABLE_INTERRUPTS(CLBR_ANY) 481af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath TRACE_IRQS_OFF 482af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl TI_flags(%ebp), %ecx 48388200bc28da38bcda1cb1bd218216e83b426d8a8Jaswinder Singh Rajput testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx 484af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jne syscall_exit_work 485af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl PT_EAX(%esp),%eax /* reload syscall return value */ 486af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jmp sysenter_exit 487af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#endif 488af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath 489fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 490f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge.pushsection .fixup,"ax" 491464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge2: movl $0,PT_FS(%esp) 492f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge jmp 1b 493f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge.popsection 4946837a54dd6127f055dcb26d00fee0df05c07a674H. Peter Anvin _ASM_EXTABLE(1b,2b) 495ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo PTGS_TO_GS_EX 4960aa97fb22624f18e5925d702ab0364d3838cfd91Roland McGrathENDPROC(ia32_sysenter_target) 4971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 498a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu/* 499a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu * syscall stub including irq exit should be protected against kprobes 500a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu */ 501a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu .pushsection .kprobes.text, "ax" 5021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # system call handler stub 5031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(system_call) 504fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME # can't unwind into user space anyway 505e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 506df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %eax # save orig_eax 5071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds SAVE_ALL 5081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 509ed75e8d58010fdc06e2c3a81bfbebae92314c7e3Laurent Vivier # system call tracing in operation / emulation 51088200bc28da38bcda1cb1bd218216e83b426d8a8Jaswinder Singh Rajput testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp) 5111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz syscall_trace_entry 512303395ac3bf3e2cb488435537d416bc840438fcbH. Peter Anvin cmpl $(NR_syscalls), %eax 5131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jae syscall_badsys 5141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_call: 5151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call *sys_call_table(,%eax,4) 516eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl %eax,PT_EAX(%esp) # store the return value 5171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_exit: 518c7e872e7da5514d014707a407ea562d197cc0136Peter Zijlstra LOCKDEP_SYS_EXIT 519139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 5201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # setting need_resched or sigpending 5211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # between sampling and the iret 52255f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 5231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx 52488200bc28da38bcda1cb1bd218216e83b426d8a8Jaswinder Singh Rajput testl $_TIF_ALLWORK_MASK, %ecx # current->work 5251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne syscall_exit_work 5261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsrestore_all: 5282e04bc76560decd9270be2a805927316f617ef56Alexander van Heukelum TRACE_IRQS_IRET 5292e04bc76560decd9270be2a805927316f617ef56Alexander van Heukelumrestore_all_notrace: 53034273f41d57ee8d854dcd2a1d754cbb546cb548fH. Peter Anvin#ifdef CONFIG_X86_ESPFIX32 531eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS 532eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge # Warning: PT_OLDSS(%esp) contains the wrong/random values if we 5335df240826c90afdc7956f55a004ea6b702df9203Stas Sergeev # are returning to the kernel. 5345df240826c90afdc7956f55a004ea6b702df9203Stas Sergeev # See comments in process.c:copy_thread() for details. 535eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movb PT_OLDSS(%esp), %ah 536eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movb PT_CS(%esp), %al 537ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax 53878be3706b21a232310590fe00258b224177ac05fRusty Russell cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax 539fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REMEMBER_STATE 5401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds je ldt_ss # returning to user-space with LDT SS 54134273f41d57ee8d854dcd2a1d754cbb546cb548fH. Peter Anvin#endif 5421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsrestore_nocheck: 543ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo RESTORE_REGS 4 # skip orig_eax/error_code 544f7f3d791e61d7baf8b0aee0384fdd469c0d2ac9bAdrian Bunkirq_return: 5453701d863b43d05ffeb223d269583398f914fb5d3Ingo Molnar INTERRUPT_RETURN 5461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.section .fixup,"ax" 54790e9f53662826db3cdd6d99bd394d727b05160c1Jeremy FitzhardingeENTRY(iret_exc) 548a879cbbb34cbecfa9707fbb6e5a00c503ac1ecb9Linus Torvalds pushl $0 # no error code 549a879cbbb34cbecfa9707fbb6e5a00c503ac1ecb9Linus Torvalds pushl $do_iret_error 550a879cbbb34cbecfa9707fbb6e5a00c503ac1ecb9Linus Torvalds jmp error_code 5511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.previous 5526837a54dd6127f055dcb26d00fee0df05c07a674H. Peter Anvin _ASM_EXTABLE(irq_return,iret_exc) 5531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 55434273f41d57ee8d854dcd2a1d754cbb546cb548fH. Peter Anvin#ifdef CONFIG_X86_ESPFIX32 555fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_RESTORE_STATE 5561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsldt_ss: 557d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell#ifdef CONFIG_PARAVIRT 558d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell /* 559d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * The kernel can't run on a non-flat stack if paravirt mode 560d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * is active. Rather than try to fixup the high bits of 561d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * ESP, bypass this code entirely. This may break DOSemu 562d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * and/or Wine support in a paravirt VM, although the option 563d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * is still available to implement the setting of the high 564d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * 16-bits in the INTERRUPT_RETURN paravirt-op. 565d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell */ 56693b1eab3d29e7ea32ee583de3362da84db06ded8Jeremy Fitzhardinge cmpl $0, pv_info+PARAVIRT_enabled 567d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell jne restore_nocheck 568d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell#endif 569d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell 570dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum/* 571dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum * Setup and switch to ESPFIX stack 572dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum * 573dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum * We're returning to userspace with a 16 bit stack. The CPU will not 574dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum * restore the high word of ESP for us on executing iret... This is an 575dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum * "official" bug of all the x86-compatible CPUs, which we can work 576dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum * around to make dosemu and wine happy. We do this by preloading the 577dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum * high word of ESP with the high word of the userspace ESP while 578dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum * compensating for the offset by changing to the ESPFIX segment with 579dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum * a base address that matches for the difference. 580dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum */ 58172c511dd596cff88d6523f231a0fbb8f73006d51Brian Gerst#define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8) 582dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum mov %esp, %edx /* load kernel esp */ 583dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum mov PT_OLDESP(%esp), %eax /* load userspace esp */ 584dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum mov %dx, %ax /* eax: new kernel esp */ 585dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum sub %eax, %edx /* offset (low word is 0) */ 586dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum shr $16, %edx 58772c511dd596cff88d6523f231a0fbb8f73006d51Brian Gerst mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */ 58872c511dd596cff88d6523f231a0fbb8f73006d51Brian Gerst mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */ 589df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $__ESPFIX_SS 590df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %eax /* new kernel esp */ 5912e04bc76560decd9270be2a805927316f617ef56Alexander van Heukelum /* Disable interrupts, but do not irqtrace this section: we 5922e04bc76560decd9270be2a805927316f617ef56Alexander van Heukelum * will soon execute iret and the tracer was already set to 5932e04bc76560decd9270be2a805927316f617ef56Alexander van Heukelum * the irqstate after the iret */ 594139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_EAX) 595dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum lss (%esp), %esp /* switch to espfix segment */ 596be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev CFI_ADJUST_CFA_OFFSET -8 597be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev jmp restore_nocheck 59834273f41d57ee8d854dcd2a1d754cbb546cb548fH. Peter Anvin#endif 599fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 60047a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichENDPROC(system_call) 6011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # perform work that needs to be done immediately before resumption 6031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 604fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_PTREGS_FRAME # can't unwind into user space anyway 6051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldswork_pending: 6061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testb $_TIF_NEED_RESCHED, %cl 6071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz work_notifysig 6081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldswork_resched: 6091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call schedule 610c7e872e7da5514d014707a407ea562d197cc0136Peter Zijlstra LOCKDEP_SYS_EXIT 611139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 6121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # setting need_resched or sigpending 6131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # between sampling and the iret 61455f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 6151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx 6161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds andl $_TIF_WORK_MASK, %ecx # is there any work to be done other 6171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # than syscall tracing? 6181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz restore_all 6191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testb $_TIF_NEED_RESCHED, %cl 6201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz work_resched 6211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldswork_notifysig: # deal with pending signals and 6231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # notify-resume requests 62474b47a7844501445d41d704fe7c626f4b1819508Joe Korty#ifdef CONFIG_VM86 625ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov testl $X86_EFLAGS_VM, PT_EFLAGS(%esp) 6261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp, %eax 6271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne work_notifysig_v86 # returning to kernel-space or 6281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # vm86-space 629969ae0bfb079de892a95aa4ca1e8ac76c4beb57cAl Viro1: 630969ae0bfb079de892a95aa4ca1e8ac76c4beb57cAl Viro#else 631969ae0bfb079de892a95aa4ca1e8ac76c4beb57cAl Viro movl %esp, %eax 632969ae0bfb079de892a95aa4ca1e8ac76c4beb57cAl Viro#endif 6333596ff4e6b2aff8a28c69af389d5046090a53330Srikar Dronamraju TRACE_IRQS_ON 6343596ff4e6b2aff8a28c69af389d5046090a53330Srikar Dronamraju ENABLE_INTERRUPTS(CLBR_NONE) 63544fbbb3dc687c9709a6f2236197316e5c79ab1ebAl Viro movb PT_CS(%esp), %bl 63644fbbb3dc687c9709a6f2236197316e5c79ab1ebAl Viro andb $SEGMENT_RPL_MASK, %bl 63744fbbb3dc687c9709a6f2236197316e5c79ab1ebAl Viro cmpb $USER_RPL, %bl 63844fbbb3dc687c9709a6f2236197316e5c79ab1ebAl Viro jb resume_kernel 6391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %edx, %edx 6401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_notify_resume 64144fbbb3dc687c9709a6f2236197316e5c79ab1ebAl Viro jmp resume_userspace 6421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 643969ae0bfb079de892a95aa4ca1e8ac76c4beb57cAl Viro#ifdef CONFIG_VM86 6441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 6451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldswork_notifysig_v86: 646df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %ecx # save ti_flags for do_notify_resume 6471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call save_v86_state # %eax contains pt_regs pointer 648df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich popl_cfi %ecx 6491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %eax, %esp 650969ae0bfb079de892a95aa4ca1e8ac76c4beb57cAl Viro jmp 1b 65174b47a7844501445d41d704fe7c626f4b1819508Joe Korty#endif 65247a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(work_pending) 6531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # perform syscall exit tracing 6551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 6561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_trace_entry: 657eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl $-ENOSYS,PT_EAX(%esp) 6581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp, %eax 659d4d67150165df8bf1cc05e532f6efca96f907cabRoland McGrath call syscall_trace_enter 660d4d67150165df8bf1cc05e532f6efca96f907cabRoland McGrath /* What it returned is what we'll actually use. */ 661303395ac3bf3e2cb488435537d416bc840438fcbH. Peter Anvin cmpl $(NR_syscalls), %eax 6621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnae syscall_call 6631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp syscall_exit 66447a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(syscall_trace_entry) 6651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # perform syscall exit tracing 6671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 6681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_exit_work: 66988200bc28da38bcda1cb1bd218216e83b426d8a8Jaswinder Singh Rajput testl $_TIF_WORK_SYSCALL_EXIT, %ecx 6701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz work_pending 67155f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_ON 672d4d67150165df8bf1cc05e532f6efca96f907cabRoland McGrath ENABLE_INTERRUPTS(CLBR_ANY) # could let syscall_trace_leave() call 6731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # schedule() instead 6741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp, %eax 675d4d67150165df8bf1cc05e532f6efca96f907cabRoland McGrath call syscall_trace_leave 6761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp resume_userspace 67747a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(syscall_exit_work) 678fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 6791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 680fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME # can't unwind into user space anyway 6811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_fault: 682e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 6831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 684eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl $-EFAULT,PT_EAX(%esp) 6851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp resume_userspace 68647a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(syscall_fault) 6871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_badsys: 689eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl $-ENOSYS,PT_EAX(%esp) 6901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp resume_userspace 69147a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(syscall_badsys) 692fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 693a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu/* 694a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu * End of kprobes section 695a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu */ 696a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu .popsection 6971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 698f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro FIXUP_ESPFIX_STACK 699dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum/* 700dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum * Switch back for ESPFIX stack to the normal zerobased stack 701dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum * 702dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum * We can't call C functions using the ESPFIX stack. This code reads 703dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum * the high word of the segment base from the GDT and swiches to the 704dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum * normal stack and adjusts ESP with the matching offset. 705dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum */ 70634273f41d57ee8d854dcd2a1d754cbb546cb548fH. Peter Anvin#ifdef CONFIG_X86_ESPFIX32 707dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum /* fixup the stack */ 70872c511dd596cff88d6523f231a0fbb8f73006d51Brian Gerst mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */ 70972c511dd596cff88d6523f231a0fbb8f73006d51Brian Gerst mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */ 710dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum shl $16, %eax 711dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum addl %esp, %eax /* the adjusted stack pointer */ 712df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $__KERNEL_DS 713df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %eax 714dc4c2a0aed3b09f6e255bd5c3faa50fe6e0b2dedAlexander van Heukelum lss (%esp), %esp /* switch to the normal stack segment */ 715f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -8 71634273f41d57ee8d854dcd2a1d754cbb546cb548fH. Peter Anvin#endif 717f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 718f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro UNWIND_ESPFIX_STACK 71934273f41d57ee8d854dcd2a1d754cbb546cb548fH. Peter Anvin#ifdef CONFIG_X86_ESPFIX32 720f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl %ss, %eax 721f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /* see if on espfix stack */ 722f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo cmpw $__ESPFIX_SS, %ax 723f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo jne 27f 724f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl $__KERNEL_DS, %eax 725f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl %eax, %ds 726f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl %eax, %es 727f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /* switch to normal stack */ 728f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo FIXUP_ESPFIX_STACK 729f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo27: 73034273f41d57ee8d854dcd2a1d754cbb546cb548fH. Peter Anvin#endif 731f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 7321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 734b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin * Build the entry stubs and pointer table with some assembler magic. 735b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin * We pack 7 stubs into a single 32-byte chunk, which will fit in a 736b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin * single cache line on all modern x86 implementations. 7371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 7384687518c4cb7807fbeff21770e309080f9eb7f2fH. Peter Anvin.section .init.rodata,"a" 7391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(interrupt) 740ea7145477a461e09d8d194cac4b996dc4f449107Jiri Olsa.section .entry.text, "ax" 741b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .p2align 5 742b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .p2align CONFIG_X86_L1_CACHE_SHIFT 7431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(irq_entries_start) 744fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 7454687518c4cb7807fbeff21770e309080f9eb7f2fH. Peter Anvinvector=FIRST_EXTERNAL_VECTOR 746b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin.rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7 747b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .balign 32 748b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .rept 7 749b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .if vector < NR_VECTORS 7508665596ec05498525014436520b316ba174a068aH. Peter Anvin .if vector <> FIRST_EXTERNAL_VECTOR 751fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 752b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .endif 753df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich1: pushl_cfi $(~vector+0x80) /* Note: always in signed byte range */ 7548665596ec05498525014436520b316ba174a068aH. Peter Anvin .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6 755b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin jmp 2f 756b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .endif 757b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .previous 7581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .long 1b 759ea7145477a461e09d8d194cac4b996dc4f449107Jiri Olsa .section .entry.text, "ax" 7601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvector=vector+1 761b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .endif 762b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .endr 763b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin2: jmp common_interrupt 7641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.endr 76547a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(irq_entries_start) 76647a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich 76747a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich.previous 76847a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(interrupt) 76947a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich.previous 7701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 77155f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar/* 77255f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar * the CPU automatically disables interrupts when executing an IRQ vector, 77355f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar * so IRQ-flags tracing has to follow that: 77455f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar */ 775b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .p2align CONFIG_X86_L1_CACHE_SHIFT 7761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldscommon_interrupt: 777e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 778b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin addl $-0x80,(%esp) /* Adjust vector into the [-256,-1] range */ 7791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds SAVE_ALL 78055f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 7811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp,%eax 7821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_IRQ 7831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp ret_from_intr 78447a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichENDPROC(common_interrupt) 785fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 7861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 787a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu/* 788a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu * Irq entries should be protected against kprobes 789a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu */ 790a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu .pushsection .kprobes.text, "ax" 79102cf94c370e0dc9bf408fe45eb86fe9ad58eaf7fTejun Heo#define BUILD_INTERRUPT3(name, nr, fn) \ 7921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(name) \ 793fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME; \ 794e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC; \ 795df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $~(nr); \ 796fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich SAVE_ALL; \ 79755f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF \ 7981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp,%eax; \ 79902cf94c370e0dc9bf408fe45eb86fe9ad58eaf7fTejun Heo call fn; \ 80055f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar jmp ret_from_intr; \ 80147a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich CFI_ENDPROC; \ 80247a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichENDPROC(name) 8031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 804cf910e83ae23692fdeefc7e506e504c4c468d38aSeiji Aguchi 805cf910e83ae23692fdeefc7e506e504c4c468d38aSeiji Aguchi#ifdef CONFIG_TRACING 806cf910e83ae23692fdeefc7e506e504c4c468d38aSeiji Aguchi#define TRACE_BUILD_INTERRUPT(name, nr) \ 807cf910e83ae23692fdeefc7e506e504c4c468d38aSeiji Aguchi BUILD_INTERRUPT3(trace_##name, nr, smp_trace_##name) 808cf910e83ae23692fdeefc7e506e504c4c468d38aSeiji Aguchi#else 809cf910e83ae23692fdeefc7e506e504c4c468d38aSeiji Aguchi#define TRACE_BUILD_INTERRUPT(name, nr) 810cf910e83ae23692fdeefc7e506e504c4c468d38aSeiji Aguchi#endif 811cf910e83ae23692fdeefc7e506e504c4c468d38aSeiji Aguchi 812cf910e83ae23692fdeefc7e506e504c4c468d38aSeiji Aguchi#define BUILD_INTERRUPT(name, nr) \ 813cf910e83ae23692fdeefc7e506e504c4c468d38aSeiji Aguchi BUILD_INTERRUPT3(name, nr, smp_##name); \ 814cf910e83ae23692fdeefc7e506e504c4c468d38aSeiji Aguchi TRACE_BUILD_INTERRUPT(name, nr) 81502cf94c370e0dc9bf408fe45eb86fe9ad58eaf7fTejun Heo 8161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* The include is where all of the SMP etc. interrupts come from */ 8171164dd0099c0d79146a55319670f57ab7ad1d352Ingo Molnar#include <asm/entry_arch.h> 8181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(coprocessor_error) 820fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 821e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 822df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $0 823df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $do_coprocessor_error 8241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 825fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 82647a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(coprocessor_error) 8271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(simd_coprocessor_error) 829fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 830e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 831df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $0 83240d2e76315da38993129090dc5d56377e573c312Brian Gerst#ifdef CONFIG_X86_INVD_BUG 83340d2e76315da38993129090dc5d56377e573c312Brian Gerst /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */ 834df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich661: pushl_cfi $do_general_protection 83540d2e76315da38993129090dc5d56377e573c312Brian Gerst662: 83640d2e76315da38993129090dc5d56377e573c312Brian Gerst.section .altinstructions,"a" 837b4ca46e4e82a0a5976fe5eab85be585d75f8202fAndy Lutomirski altinstruction_entry 661b, 663f, X86_FEATURE_XMM, 662b-661b, 664f-663f 83840d2e76315da38993129090dc5d56377e573c312Brian Gerst.previous 83940d2e76315da38993129090dc5d56377e573c312Brian Gerst.section .altinstr_replacement,"ax" 84040d2e76315da38993129090dc5d56377e573c312Brian Gerst663: pushl $do_simd_coprocessor_error 84140d2e76315da38993129090dc5d56377e573c312Brian Gerst664: 84240d2e76315da38993129090dc5d56377e573c312Brian Gerst.previous 84340d2e76315da38993129090dc5d56377e573c312Brian Gerst#else 844df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $do_simd_coprocessor_error 84540d2e76315da38993129090dc5d56377e573c312Brian Gerst#endif 8461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 847fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 84847a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(simd_coprocessor_error) 8491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(device_not_available) 851fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 852e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 853df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $-1 # mark this as an int 854df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $do_device_not_available 8557643e9b936b4af31ba4851eb7d5b3a3bfad52502Alexander van Heukelum jmp error_code 856fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 85747a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(device_not_available) 8581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 859d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell#ifdef CONFIG_PARAVIRT 860d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty RussellENTRY(native_iret) 8613701d863b43d05ffeb223d269583398f914fb5d3Ingo Molnar iret 8626837a54dd6127f055dcb26d00fee0df05c07a674H. Peter Anvin _ASM_EXTABLE(native_iret, iret_exc) 86347a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(native_iret) 864d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell 865d75cd22fdd5f7d203fb60014d426942df33dd9a6Jeremy FitzhardingeENTRY(native_irq_enable_sysexit) 866d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell sti 867d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell sysexit 868d75cd22fdd5f7d203fb60014d426942df33dd9a6Jeremy FitzhardingeEND(native_irq_enable_sysexit) 869d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell#endif 870d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell 8711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(overflow) 872fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 873e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 874df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $0 875df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $do_overflow 8761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 877fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 87847a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(overflow) 8791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(bounds) 881fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 882e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 883df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $0 884df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $do_bounds 8851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 886fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 88747a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(bounds) 8881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(invalid_op) 890fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 891e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 892df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $0 893df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $do_invalid_op 8941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 895fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 89647a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(invalid_op) 8971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(coprocessor_segment_overrun) 899fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 900e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 901df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $0 902df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $do_coprocessor_segment_overrun 9031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 904fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 90547a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(coprocessor_segment_overrun) 9061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(invalid_TSS) 908fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_EC_FRAME 909e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 910df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $do_invalid_TSS 9111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 912fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 91347a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(invalid_TSS) 9141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(segment_not_present) 916fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_EC_FRAME 917e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 918df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $do_segment_not_present 9191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 920fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 92147a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(segment_not_present) 9221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(stack_segment) 924fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_EC_FRAME 925e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 926df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $do_stack_segment 9271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 928fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 92947a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(stack_segment) 9301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(alignment_check) 932fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_EC_FRAME 933e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 934df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $do_alignment_check 9351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 936fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 93747a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(alignment_check) 9381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 939d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.PENTRY(divide_error) 940d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P RING0_INT_FRAME 941e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 942df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $0 # no error code 943df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $do_divide_error 9441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 945fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 94647a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(divide_error) 9471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef CONFIG_X86_MCE 9491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(machine_check) 950fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 951e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 952df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $0 953df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi machine_check_vector 9541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 955fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 95647a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(machine_check) 9571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 9581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(spurious_interrupt_bug) 960fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 961e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 962df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $0 963df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $do_spurious_interrupt_bug 9641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 965fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 96647a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(spurious_interrupt_bug) 967a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu/* 968a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu * End of kprobes section 969a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu */ 970a00e817f42663941ea0aa5f85a9d1c4f8b212839Masami Hiramatsu .popsection 9711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9725ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge#ifdef CONFIG_XEN 973e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge/* Xen doesn't set %esp to be precisely what the normal sysenter 974e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge entrypoint expects, so fix it up before using the normal path. */ 975e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy FitzhardingeENTRY(xen_sysenter_target) 976e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge RING0_INT_FRAME 977e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge addl $5*4, %esp /* remove xen-provided frame */ 9782ddf9b7b3e6660199269e34cfa27148440ddc3bfJan Beulich CFI_ADJUST_CFA_OFFSET -5*4 979e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge jmp sysenter_past_esp 980557d7d4e294ee6fb1db0cb6c1ec97a1c908b880dGlauber Costa CFI_ENDPROC 981e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge 9825ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy FitzhardingeENTRY(xen_hypervisor_callback) 9835ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_STARTPROC 984a349e23d1cf746f8bdc603dcc61fae9ee4a695f6David Vrabel pushl_cfi $-1 /* orig_ax = -1 => not a system call */ 9855ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge SAVE_ALL 9865ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge TRACE_IRQS_OFF 9879ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge 9889ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge /* Check to see if we got the event in the critical 9899ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge region in xen_iret_direct, after we've reenabled 9909ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge events and checked for pending events. This simulates 9919ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge iret instruction's behaviour where it delivers a 9929ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge pending interrupt when enabling interrupts. */ 9939ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge movl PT_EIP(%esp),%eax 9949ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge cmpl $xen_iret_start_crit,%eax 9959ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge jb 1f 9969ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge cmpl $xen_iret_end_crit,%eax 9979ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge jae 1f 9989ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge 9990f2c87695219b1129ccf93e0f58acdcdd49724b9Jeremy Fitzhardinge jmp xen_iret_crit_fixup 1000e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge 1001e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy FitzhardingeENTRY(xen_do_upcall) 1002b77797fb2bf31bf076e6b69736119bc6a077525bJeremy Fitzhardinge1: mov %esp, %eax 10035ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge call xen_evtchn_do_upcall 10045ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp ret_from_intr 10055ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ENDPROC 10065ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy FitzhardingeENDPROC(xen_hypervisor_callback) 10075ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge 10085ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# Hypervisor uses this for application faults while it executes. 10095ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# We get here for two reasons: 10105ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# 1. Fault while reloading DS, ES, FS or GS 10115ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# 2. Fault while executing IRET 10125ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# Category 1 we fix up by reattempting the load, and zeroing the segment 10135ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# register if the load fails. 10145ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# Category 2 we fix up by jumping to do_iret_error. We cannot use the 10155ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# normal Linux return path in this case because if we use the IRET hypercall 10165ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# to pop the stack frame we end up in an infinite loop of failsafe callbacks. 10175ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# We distinguish between categories by maintaining a status value in EAX. 10185ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy FitzhardingeENTRY(xen_failsafe_callback) 10195ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_STARTPROC 1020df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %eax 10215ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl $1,%eax 10225ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge1: mov 4(%esp),%ds 10235ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge2: mov 8(%esp),%es 10245ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge3: mov 12(%esp),%fs 10255ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge4: mov 16(%esp),%gs 1026a349e23d1cf746f8bdc603dcc61fae9ee4a695f6David Vrabel /* EAX == 0 => Category 1 (Bad segment) 1027a349e23d1cf746f8bdc603dcc61fae9ee4a695f6David Vrabel EAX != 0 => Category 2 (Bad IRET) */ 10285ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge testl %eax,%eax 1029df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich popl_cfi %eax 10305ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge lea 16(%esp),%esp 10315ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET -16 10325ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jz 5f 1033a349e23d1cf746f8bdc603dcc61fae9ee4a695f6David Vrabel jmp iret_exc 1034a349e23d1cf746f8bdc603dcc61fae9ee4a695f6David Vrabel5: pushl_cfi $-1 /* orig_ax = -1 => not a system call */ 10355ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge SAVE_ALL 10365ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp ret_from_exception 10375ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ENDPROC 10385ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge 10395ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge.section .fixup,"ax" 10405ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge6: xorl %eax,%eax 10415ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl %eax,4(%esp) 10425ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp 1b 10435ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge7: xorl %eax,%eax 10445ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl %eax,8(%esp) 10455ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp 2b 10465ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge8: xorl %eax,%eax 10475ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl %eax,12(%esp) 10485ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp 3b 10495ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge9: xorl %eax,%eax 10505ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl %eax,16(%esp) 10515ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp 4b 10525ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge.previous 10536837a54dd6127f055dcb26d00fee0df05c07a674H. Peter Anvin _ASM_EXTABLE(1b,6b) 10546837a54dd6127f055dcb26d00fee0df05c07a674H. Peter Anvin _ASM_EXTABLE(2b,7b) 10556837a54dd6127f055dcb26d00fee0df05c07a674H. Peter Anvin _ASM_EXTABLE(3b,8b) 10566837a54dd6127f055dcb26d00fee0df05c07a674H. Peter Anvin _ASM_EXTABLE(4b,9b) 10575ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy FitzhardingeENDPROC(xen_failsafe_callback) 10585ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge 1059bc2b0331e077f576369a2b6c75d15ed4de4ef91fK. Y. SrinivasanBUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR, 106038e20b07efd541a959de367dc90a17f92ce2e8a6Sheng Yang xen_evtchn_do_upcall) 106138e20b07efd541a959de367dc90a17f92ce2e8a6Sheng Yang 10625ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge#endif /* CONFIG_XEN */ 1063bc2b0331e077f576369a2b6c75d15ed4de4ef91fK. Y. Srinivasan 1064bc2b0331e077f576369a2b6c75d15ed4de4ef91fK. Y. Srinivasan#if IS_ENABLED(CONFIG_HYPERV) 1065bc2b0331e077f576369a2b6c75d15ed4de4ef91fK. Y. Srinivasan 1066bc2b0331e077f576369a2b6c75d15ed4de4ef91fK. Y. SrinivasanBUILD_INTERRUPT3(hyperv_callback_vector, HYPERVISOR_CALLBACK_VECTOR, 1067bc2b0331e077f576369a2b6c75d15ed4de4ef91fK. Y. Srinivasan hyperv_vector_handler) 1068bc2b0331e077f576369a2b6c75d15ed4de4ef91fK. Y. Srinivasan 1069bc2b0331e077f576369a2b6c75d15ed4de4ef91fK. Y. Srinivasan#endif /* CONFIG_HYPERV */ 10705ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge 1071606576ce816603d9fe1fb453a88bc6eea16ca709Steven Rostedt#ifdef CONFIG_FUNCTION_TRACER 1072d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt#ifdef CONFIG_DYNAMIC_FTRACE 1073d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 1074d61f82d06672f57fca410da6f7fffd15867db622Steven RostedtENTRY(mcount) 1075d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt ret 1076d61f82d06672f57fca410da6f7fffd15867db622Steven RostedtEND(mcount) 1077d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 1078d61f82d06672f57fca410da6f7fffd15867db622Steven RostedtENTRY(ftrace_caller) 107960a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt cmpl $0, function_trace_stop 108060a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt jne ftrace_stub 108160a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt 1082d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt pushl %eax 1083d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt pushl %ecx 1084d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt pushl %edx 108508f6fba503111e0336f2b4d6915a4a18f9b60e51Steven Rostedt pushl $0 /* Pass NULL as regs pointer */ 108608f6fba503111e0336f2b4d6915a4a18f9b60e51Steven Rostedt movl 4*4(%esp), %eax 1087d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt movl 0x4(%ebp), %edx 10881739f09e33d8f66bf48ddbc3eca615574da6c4f6Steven Rostedt movl function_trace_op, %ecx 1089395a59d0f8e86bb39cd700c3d185d30c670bb958Abhishek Sagar subl $MCOUNT_INSN_SIZE, %eax 1090d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 1091d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt.globl ftrace_call 1092d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedtftrace_call: 1093d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt call ftrace_stub 1094d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 109508f6fba503111e0336f2b4d6915a4a18f9b60e51Steven Rostedt addl $4,%esp /* skip NULL pointer */ 1096d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt popl %edx 1097d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt popl %ecx 1098d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt popl %eax 10994de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedtftrace_ret: 11005a45cfe1c64862e8cd3b0d79d7c4ba71c3118915Steven Rostedt#ifdef CONFIG_FUNCTION_GRAPH_TRACER 11015a45cfe1c64862e8cd3b0d79d7c4ba71c3118915Steven Rostedt.globl ftrace_graph_call 11025a45cfe1c64862e8cd3b0d79d7c4ba71c3118915Steven Rostedtftrace_graph_call: 11035a45cfe1c64862e8cd3b0d79d7c4ba71c3118915Steven Rostedt jmp ftrace_stub 11045a45cfe1c64862e8cd3b0d79d7c4ba71c3118915Steven Rostedt#endif 1105d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 1106d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt.globl ftrace_stub 1107d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedtftrace_stub: 1108d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt ret 1109d61f82d06672f57fca410da6f7fffd15867db622Steven RostedtEND(ftrace_caller) 1110d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 11114de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven RostedtENTRY(ftrace_regs_caller) 11124de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt pushf /* push flags before compare (in cs location) */ 11134de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt cmpl $0, function_trace_stop 11144de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt jne ftrace_restore_flags 11154de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt 11164de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt /* 11174de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt * i386 does not save SS and ESP when coming from kernel. 11184de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt * Instead, to get sp, ®s->sp is used (see ptrace.h). 11194de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt * Unfortunately, that means eflags must be at the same location 11204de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt * as the current return ip is. We move the return ip into the 11214de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt * ip location, and move flags into the return ip location. 11224de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt */ 11234de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt pushl 4(%esp) /* save return ip into ip slot */ 11244de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt 11254de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt pushl $0 /* Load 0 into orig_ax */ 11264de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt pushl %gs 11274de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt pushl %fs 11284de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt pushl %es 11294de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt pushl %ds 11304de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt pushl %eax 11314de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt pushl %ebp 11324de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt pushl %edi 11334de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt pushl %esi 11344de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt pushl %edx 11354de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt pushl %ecx 11364de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt pushl %ebx 11374de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt 11384de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt movl 13*4(%esp), %eax /* Get the saved flags */ 11394de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt movl %eax, 14*4(%esp) /* Move saved flags into regs->flags location */ 11404de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt /* clobbering return ip */ 11414de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt movl $__KERNEL_CS,13*4(%esp) 11424de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt 11434de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt movl 12*4(%esp), %eax /* Load ip (1st parameter) */ 1144a5e37863ab31d78faddff15675c89979792bc0bdMasami Hiramatsu subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */ 1145e4ea3f6b1bf3d489674a3660db652636e50186f9Uros Bizjak movl 0x4(%ebp), %edx /* Load parent ip (2nd parameter) */ 11461739f09e33d8f66bf48ddbc3eca615574da6c4f6Steven Rostedt movl function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */ 1147e4ea3f6b1bf3d489674a3660db652636e50186f9Uros Bizjak pushl %esp /* Save pt_regs as 4th parameter */ 11484de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt 11494de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven RostedtGLOBAL(ftrace_regs_call) 11504de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt call ftrace_stub 11514de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt 11524de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt addl $4, %esp /* Skip pt_regs */ 11534de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt movl 14*4(%esp), %eax /* Move flags back into cs */ 11544de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt movl %eax, 13*4(%esp) /* Needed to keep addl from modifying flags */ 11554de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt movl 12*4(%esp), %eax /* Get return ip from regs->ip */ 11564de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt movl %eax, 14*4(%esp) /* Put return ip back for ret */ 11574de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt 11584de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt popl %ebx 11594de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt popl %ecx 11604de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt popl %edx 11614de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt popl %esi 11624de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt popl %edi 11634de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt popl %ebp 11644de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt popl %eax 11654de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt popl %ds 11664de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt popl %es 11674de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt popl %fs 11684de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt popl %gs 11694de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt addl $8, %esp /* Skip orig_ax and ip */ 11704de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt popf /* Pop flags at end (no addl to corrupt flags) */ 11714de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt jmp ftrace_ret 11724de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt 11734de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedtftrace_restore_flags: 11744de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt popf 11754de72395ff4cf48e23b61986dbc90b99a7c4ed97Steven Rostedt jmp ftrace_stub 1176d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt#else /* ! CONFIG_DYNAMIC_FTRACE */ 1177d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 117816444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de MeloENTRY(mcount) 1179af058ab04d3d11383e5159132fc78a3700be3af5H. Peter Anvin cmpl $__PAGE_OFFSET, %esp 1180af058ab04d3d11383e5159132fc78a3700be3af5H. Peter Anvin jb ftrace_stub /* Paging not enabled yet? */ 1181af058ab04d3d11383e5159132fc78a3700be3af5H. Peter Anvin 118260a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt cmpl $0, function_trace_stop 118360a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt jne ftrace_stub 118460a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt 118516444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo cmpl $ftrace_stub, ftrace_trace_function 118616444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo jnz trace 1187fb52607afcd0629776f1dc9e657647ceae81dd50Frederic Weisbecker#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1188c2324b694fa8ffee382a124198c68754088e483cIngo Molnar cmpl $ftrace_stub, ftrace_graph_return 1189fb52607afcd0629776f1dc9e657647ceae81dd50Frederic Weisbecker jnz ftrace_graph_caller 1190e49dc19c6a19ea112fcb94b7c62ec62cdd5c08aaSteven Rostedt 1191e49dc19c6a19ea112fcb94b7c62ec62cdd5c08aaSteven Rostedt cmpl $ftrace_graph_entry_stub, ftrace_graph_entry 1192e49dc19c6a19ea112fcb94b7c62ec62cdd5c08aaSteven Rostedt jnz ftrace_graph_caller 1193caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker#endif 119416444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo.globl ftrace_stub 119516444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Meloftrace_stub: 119616444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo ret 119716444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo 119816444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo /* taken from glibc */ 119916444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melotrace: 120016444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo pushl %eax 120116444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo pushl %ecx 120216444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo pushl %edx 120316444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo movl 0xc(%esp), %eax 120416444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo movl 0x4(%ebp), %edx 1205395a59d0f8e86bb39cd700c3d185d30c670bb958Abhishek Sagar subl $MCOUNT_INSN_SIZE, %eax 120616444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo 1207d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt call *ftrace_trace_function 120816444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo 120916444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo popl %edx 121016444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo popl %ecx 121116444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo popl %eax 121216444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo jmp ftrace_stub 121316444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de MeloEND(mcount) 1214d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt#endif /* CONFIG_DYNAMIC_FTRACE */ 1215606576ce816603d9fe1fb453a88bc6eea16ca709Steven Rostedt#endif /* CONFIG_FUNCTION_TRACER */ 121616444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo 1217fb52607afcd0629776f1dc9e657647ceae81dd50Frederic Weisbecker#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1218fb52607afcd0629776f1dc9e657647ceae81dd50Frederic WeisbeckerENTRY(ftrace_graph_caller) 1219caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %eax 1220caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %ecx 1221caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %edx 12221dc1c6adf38bc5799d1594681645ced40ced4b6bFrederic Weisbecker movl 0xc(%esp), %edx 1223caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker lea 0x4(%ebp), %eax 122471e308a239c098673570d0b417d42262bb535909Steven Rostedt movl (%ebp), %ecx 1225bb4304c71c97bf727ec43cd2f195c2c237c27fd3Steven Rostedt subl $MCOUNT_INSN_SIZE, %edx 1226caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker call prepare_ftrace_return 1227caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %edx 1228caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %ecx 1229caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %eax 1230e7d3737ea1b102030f44e96c97754101e41515f0Frederic Weisbecker ret 1231fb52607afcd0629776f1dc9e657647ceae81dd50Frederic WeisbeckerEND(ftrace_graph_caller) 1232caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker 1233caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker.globl return_to_handler 1234caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbeckerreturn_to_handler: 1235caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %eax 1236caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %edx 123771e308a239c098673570d0b417d42262bb535909Steven Rostedt movl %ebp, %eax 1238caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker call ftrace_return_to_handler 1239194ec34184869f0de1cf255c924fc5299e1b3d27Steven Rostedt movl %eax, %ecx 1240caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %edx 1241caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %eax 1242194ec34184869f0de1cf255c924fc5299e1b3d27Steven Rostedt jmp *%ecx 1243e7d3737ea1b102030f44e96c97754101e41515f0Frederic Weisbecker#endif 124416444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo 1245d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum/* 1246d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * Some functions should be protected against kprobes 1247d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum */ 1248d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum .pushsection .kprobes.text, "ax" 1249d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 125025c74b10bacead867478480170083f69cfc0db48Seiji Aguchi#ifdef CONFIG_TRACING 125125c74b10bacead867478480170083f69cfc0db48Seiji AguchiENTRY(trace_page_fault) 125225c74b10bacead867478480170083f69cfc0db48Seiji Aguchi RING0_EC_FRAME 125325c74b10bacead867478480170083f69cfc0db48Seiji Aguchi ASM_CLAC 125425c74b10bacead867478480170083f69cfc0db48Seiji Aguchi pushl_cfi $trace_do_page_fault 125525c74b10bacead867478480170083f69cfc0db48Seiji Aguchi jmp error_code 125625c74b10bacead867478480170083f69cfc0db48Seiji Aguchi CFI_ENDPROC 125725c74b10bacead867478480170083f69cfc0db48Seiji AguchiEND(trace_page_fault) 125825c74b10bacead867478480170083f69cfc0db48Seiji Aguchi#endif 125925c74b10bacead867478480170083f69cfc0db48Seiji Aguchi 1260d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumENTRY(page_fault) 1261d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum RING0_EC_FRAME 1262e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 1263df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $do_page_fault 1264d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum ALIGN 1265d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelumerror_code: 1266ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo /* the function address is in %gs's slot on the stack */ 1267df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %fs 1268ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo /*CFI_REL_OFFSET fs, 0*/ 1269df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %es 1270d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /*CFI_REL_OFFSET es, 0*/ 1271df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %ds 1272d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /*CFI_REL_OFFSET ds, 0*/ 1273df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %eax 1274d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET eax, 0 1275df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %ebp 1276d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET ebp, 0 1277df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %edi 1278d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET edi, 0 1279df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %esi 1280d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET esi, 0 1281df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %edx 1282d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET edx, 0 1283df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %ecx 1284d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET ecx, 0 1285df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %ebx 1286d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET ebx, 0 1287d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cld 1288d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl $(__KERNEL_PERCPU), %ecx 1289d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %ecx, %fs 1290d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum UNWIND_ESPFIX_STACK 1291ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo GS_TO_REG %ecx 1292ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo movl PT_GS(%esp), %edi # get the function address 1293d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl PT_ORIG_EAX(%esp), %edx # get the error code 1294d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart 1295ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo REG_TO_PTGS %ecx 1296ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo SET_KERNEL_GS %ecx 1297d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl $(__USER_DS), %ecx 1298d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %ecx, %ds 1299d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %ecx, %es 1300d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum TRACE_IRQS_OFF 1301d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %esp,%eax # pt_regs pointer 1302d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum call *%edi 1303d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp ret_from_exception 1304d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ENDPROC 1305d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumEND(page_fault) 1306d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1307d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum/* 1308d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * Debug traps and NMI can happen at the one SYSENTER instruction 1309d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * that sets up the real kernel stack. Check here, since we can't 1310d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * allow the wrong stack to be used. 1311d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * 1312d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have 1313d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * already pushed 3 words if it hits on the sysenter instruction: 1314d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * eflags, cs and eip. 1315d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * 1316d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * We just load the right stack, and push the three (known) values 1317d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * by hand onto the new stack - while updating the return eip past 1318d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * the instruction that would have done it for sysenter. 1319d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum */ 1320f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro FIX_STACK offset ok label 1321f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo cmpw $__KERNEL_CS, 4(%esp) 1322f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo jne \ok 1323f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo\label: 1324f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl TSS_sysenter_sp0 + \offset(%esp), %esp 1325f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_DEF_CFA esp, 0 1326f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_UNDEFINED eip 1327df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushfl_cfi 1328df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $__KERNEL_CS 1329df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $sysenter_past_esp 1330d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET eip, 0 1331f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 1332d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1333d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumENTRY(debug) 1334d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum RING0_INT_FRAME 1335e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 1336d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpl $ia32_sysenter_target,(%esp) 1337d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jne debug_stack_correct 1338f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn 1339d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelumdebug_stack_correct: 1340df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $-1 # mark this as an int 1341d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum SAVE_ALL 1342d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum TRACE_IRQS_OFF 1343d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum xorl %edx,%edx # error code 0 1344d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %esp,%eax # pt_regs pointer 1345d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum call do_debug 1346d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp ret_from_exception 1347d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ENDPROC 1348d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumEND(debug) 1349d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1350d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum/* 1351d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * NMI is doubly nasty. It can happen _while_ we're handling 1352d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * a debug fault, and the debug fault hasn't yet been able to 1353d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * clear up the stack. So we first check whether we got an 1354d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * NMI on the sysenter entry path, but after that we need to 1355d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * check whether we got an NMI on the debug path where the debug 1356d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * fault happened on the sysenter path. 1357d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum */ 1358d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumENTRY(nmi) 1359d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum RING0_INT_FRAME 1360e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 136134273f41d57ee8d854dcd2a1d754cbb546cb548fH. Peter Anvin#ifdef CONFIG_X86_ESPFIX32 1362df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %eax 1363d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %ss, %eax 1364d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpw $__ESPFIX_SS, %ax 1365df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich popl_cfi %eax 1366d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum je nmi_espfix_stack 136734273f41d57ee8d854dcd2a1d754cbb546cb548fH. Peter Anvin#endif 1368d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpl $ia32_sysenter_target,(%esp) 1369d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum je nmi_stack_fixup 1370df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %eax 1371d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %esp,%eax 1372d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /* Do not access memory above the end of our stack page, 1373d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * it might not exist. 1374d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum */ 1375d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum andl $(THREAD_SIZE-1),%eax 1376d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpl $(THREAD_SIZE-20),%eax 1377df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich popl_cfi %eax 1378d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jae nmi_stack_correct 1379d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpl $ia32_sysenter_target,12(%esp) 1380d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum je nmi_debug_stack_check 1381d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelumnmi_stack_correct: 1382d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /* We have a RING0_INT_FRAME here */ 1383df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %eax 1384d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum SAVE_ALL 1385d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum xorl %edx,%edx # zero error code 1386d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %esp,%eax # pt_regs pointer 1387d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum call do_nmi 13882e04bc76560decd9270be2a805927316f617ef56Alexander van Heukelum jmp restore_all_notrace 1389d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ENDPROC 1390d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1391d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelumnmi_stack_fixup: 1392d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum RING0_INT_FRAME 1393f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo FIX_STACK 12, nmi_stack_correct, 1 1394d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp nmi_stack_correct 1395d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1396d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelumnmi_debug_stack_check: 1397d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /* We have a RING0_INT_FRAME here */ 1398d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpw $__KERNEL_CS,16(%esp) 1399d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jne nmi_stack_correct 1400d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpl $debug,(%esp) 1401d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jb nmi_stack_correct 1402d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpl $debug_esp_fix_insn,(%esp) 1403d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum ja nmi_stack_correct 1404f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo FIX_STACK 24, nmi_stack_correct, 1 1405d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp nmi_stack_correct 1406d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 140734273f41d57ee8d854dcd2a1d754cbb546cb548fH. Peter Anvin#ifdef CONFIG_X86_ESPFIX32 1408d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelumnmi_espfix_stack: 1409d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /* We have a RING0_INT_FRAME here. 1410d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * 1411d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * create the pointer to lss back 1412d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum */ 1413df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %ss 1414df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %esp 1415bda3a89745d7bb88d3e735046c0cdc3d0eb2ac24Stas Sergeev addl $4, (%esp) 1416d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /* copy the iret frame of 12 bytes */ 1417d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum .rept 3 1418df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi 16(%esp) 1419d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum .endr 1420df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi %eax 1421d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum SAVE_ALL 1422d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum FIXUP_ESPFIX_STACK # %eax == %esp 1423d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum xorl %edx,%edx # zero error code 1424d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum call do_nmi 1425d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum RESTORE_REGS 1426d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum lss 12+4(%esp), %esp # back to espfix stack 1427d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET -24 1428d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp irq_return 142934273f41d57ee8d854dcd2a1d754cbb546cb548fH. Peter Anvin#endif 1430d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ENDPROC 1431d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumEND(nmi) 1432d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1433d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumENTRY(int3) 1434d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum RING0_INT_FRAME 1435e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 1436df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $-1 # mark this as an int 1437d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum SAVE_ALL 1438d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum TRACE_IRQS_OFF 1439d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum xorl %edx,%edx # zero error code 1440d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %esp,%eax # pt_regs pointer 1441d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum call do_int3 1442d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp ret_from_exception 1443d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ENDPROC 1444d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumEND(int3) 1445d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1446d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumENTRY(general_protection) 1447d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum RING0_EC_FRAME 1448df5d1874ce1a1f0e0eceff4fa3a9d45620243a68Jan Beulich pushl_cfi $do_general_protection 1449d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp error_code 1450d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ENDPROC 1451d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumEND(general_protection) 1452d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1453631bc4878220932fe67fc46fc7cf7cccdb1ec597Gleb Natapov#ifdef CONFIG_KVM_GUEST 1454631bc4878220932fe67fc46fc7cf7cccdb1ec597Gleb NatapovENTRY(async_page_fault) 1455631bc4878220932fe67fc46fc7cf7cccdb1ec597Gleb Natapov RING0_EC_FRAME 1456e59d1b0a24199db01978e6c1e89859eda93ce683H. Peter Anvin ASM_CLAC 145760cf637a13932a4750da6746efd0199e8a4c341bJan Beulich pushl_cfi $do_async_page_fault 1458631bc4878220932fe67fc46fc7cf7cccdb1ec597Gleb Natapov jmp error_code 1459631bc4878220932fe67fc46fc7cf7cccdb1ec597Gleb Natapov CFI_ENDPROC 14602ae9d293b14d17f35eff624272cfecac7979a2eeSedat DilekEND(async_page_fault) 1461631bc4878220932fe67fc46fc7cf7cccdb1ec597Gleb Natapov#endif 1462631bc4878220932fe67fc46fc7cf7cccdb1ec597Gleb Natapov 1463d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum/* 1464d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * End of kprobes section 1465d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum */ 1466d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum .popsection 1467