entry_32.S revision 88200bc28da38bcda1cb1bd218216e83b426d8a8
11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (C) 1991, 1992 Linus Torvalds 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * entry.S contains the system-call and fault low-level handling routines. 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This also contains the timer-interrupt handler, as well as all interrupts 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * and faults that can result in a task-switch. 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * NOTE: This code handles signal-recognition, which happens every time 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * after a timer-interrupt and after each system call. 131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * I changed all the .align's to 4 (16 byte alignment), as that's faster 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * on a 486. 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 17889f21ce272e38db19c8114a7e0a5793d4590077Andi Kleen * Stack layout in 'syscall_exit': 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * ptrace needs to have all regs on the stack. 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * if the order here is changed, it needs to be 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * updated in fork.c:copy_process, signal.c:do_signal, 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * ptrace.c and ptrace.h 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 0(%esp) - %ebx 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 4(%esp) - %ecx 251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 8(%esp) - %edx 261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * C(%esp) - %esi 271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 10(%esp) - %edi 281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 14(%esp) - %ebp 291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 18(%esp) - %eax 301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1C(%esp) - %ds 311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 20(%esp) - %es 32464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge * 24(%esp) - %fs 33ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS 34ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * 2C(%esp) - orig_eax 35ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * 30(%esp) - %eip 36ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * 34(%esp) - %cs 37ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * 38(%esp) - %eflags 38ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * 3C(%esp) - %oldesp 39ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * 40(%esp) - %oldss 401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * "current" is in register %ebx during any slow entries. 421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/linkage.h> 451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/thread_info.h> 4655f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar#include <asm/irqflags.h> 471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/errno.h> 481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/segment.h> 491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/smp.h> 500341c14da49e7b93d2998926f6ac89a3129e3fa1Jeremy Fitzhardinge#include <asm/page_types.h> 511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/desc.h> 52be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev#include <asm/percpu.h> 53fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich#include <asm/dwarf2.h> 54ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov#include <asm/processor-flags.h> 55395a59d0f8e86bb39cd700c3d185d30c670bb958Abhishek Sagar#include <asm/ftrace.h> 569b7dc567d03d74a1fbae84e88949b6a60d922d82Thomas Gleixner#include <asm/irq_vectors.h> 571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 58af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */ 59af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#include <linux/elf-em.h> 60af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#define AUDIT_ARCH_I386 (EM_386|__AUDIT_ARCH_LE) 61af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#define __AUDIT_ARCH_LE 0x40000000 62af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath 63af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#ifndef CONFIG_AUDITSYSCALL 64af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#define sysenter_audit syscall_trace_entry 65af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#define sysexit_audit syscall_exit_work 66af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#endif 67af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath 68139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell/* 69139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * We use macros for low-level operations which need to be overridden 70139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * for paravirtualization. The following will never clobber any registers: 71139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * INTERRUPT_RETURN (aka. "iret") 72139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") 73d75cd22fdd5f7d203fb60014d426942df33dd9a6Jeremy Fitzhardinge * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit"). 74139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * 75139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must 76139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). 77139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * Allowing a register to be clobbered can shrink the paravirt replacement 78139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * enough to patch inline, increasing performance. 79139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell */ 80139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell 811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define nr_syscalls ((syscall_table_size)/4) 821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef CONFIG_PREEMPT 84139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell#define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF 851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#else 86139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell#define preempt_stop(clobbers) 871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define resume_kernel restore_nocheck 881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9055f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar.macro TRACE_IRQS_IRET 9155f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar#ifdef CONFIG_TRACE_IRQFLAGS 92ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off? 9355f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar jz 1f 9455f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_ON 9555f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar1: 9655f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar#endif 9755f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar.endm 9855f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar 994031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov#ifdef CONFIG_VM86 1004031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov#define resume_userspace_sig check_userspace 1014031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov#else 1024031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov#define resume_userspace_sig resume_userspace 1034031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov#endif 1044031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov 105ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo/* 106ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * User gs save/restore 107ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * 108ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * %gs is used for userland TLS and kernel only uses it for stack 109ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * canary which is required to be at %gs:20 by gcc. Read the comment 110ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * at the top of stackprotector.h for more info. 111ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * 112ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo * Local labels 98 and 99 are used. 113ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo */ 114ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo#ifdef CONFIG_X86_32_LAZY_GS 115ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo 116ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo /* unfortunately push/pop can't be no-op */ 117ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro PUSH_GS 118ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo pushl $0 119ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo CFI_ADJUST_CFA_OFFSET 4 120ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 121ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro POP_GS pop=0 122ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo addl $(4 + \pop), %esp 123ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo CFI_ADJUST_CFA_OFFSET -(4 + \pop) 124ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 125ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro POP_GS_EX 126ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 127ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo 128ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo /* all the rest are no-op */ 129ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro PTGS_TO_GS 130ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 131ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro PTGS_TO_GS_EX 132ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 133ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro GS_TO_REG reg 134ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 135ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro REG_TO_PTGS reg 136ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 137ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro SET_KERNEL_GS reg 138ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 139ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo 140ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo#else /* CONFIG_X86_32_LAZY_GS */ 141ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo 142ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro PUSH_GS 143ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo pushl %gs 144ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo CFI_ADJUST_CFA_OFFSET 4 145ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo /*CFI_REL_OFFSET gs, 0*/ 146ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 147ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo 148ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro POP_GS pop=0 149ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo98: popl %gs 150ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo CFI_ADJUST_CFA_OFFSET -4 151ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo /*CFI_RESTORE gs*/ 152ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo .if \pop <> 0 153ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo add $\pop, %esp 154ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo CFI_ADJUST_CFA_OFFSET -\pop 155ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo .endif 156ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 157ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro POP_GS_EX 158ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.pushsection .fixup, "ax" 159ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo99: movl $0, (%esp) 160ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo jmp 98b 161ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.section __ex_table, "a" 162ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo .align 4 163ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo .long 98b, 99b 164ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.popsection 165ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 166ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo 167ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro PTGS_TO_GS 168ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo98: mov PT_GS(%esp), %gs 169ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 170ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro PTGS_TO_GS_EX 171ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.pushsection .fixup, "ax" 172ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo99: movl $0, PT_GS(%esp) 173ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo jmp 98b 174ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.section __ex_table, "a" 175ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo .align 4 176ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo .long 98b, 99b 177ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.popsection 178ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 179ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo 180ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro GS_TO_REG reg 181ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo movl %gs, \reg 182ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo /*CFI_REGISTER gs, \reg*/ 183ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 184ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro REG_TO_PTGS reg 185ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo movl \reg, PT_GS(%esp) 186ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo /*CFI_REL_OFFSET gs, PT_GS*/ 187ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 188ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro SET_KERNEL_GS reg 18960a5317ff0f42dd313094b88f809f63041568b08Tejun Heo movl $(__KERNEL_STACK_CANARY), \reg 190ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo movl \reg, %gs 191ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.endm 192ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo 193ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo#endif /* CONFIG_X86_32_LAZY_GS */ 194ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo 195f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro SAVE_ALL 196f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo cld 197ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo PUSH_GS 198f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl %fs 199f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 200f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_REL_OFFSET fs, 0;*/ 201f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl %es 202f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 203f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_REL_OFFSET es, 0;*/ 204f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl %ds 205f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 206f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_REL_OFFSET ds, 0;*/ 207f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl %eax 208f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 209f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_REL_OFFSET eax, 0 210f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl %ebp 211f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 212f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_REL_OFFSET ebp, 0 213f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl %edi 214f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 215f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_REL_OFFSET edi, 0 216f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl %esi 217f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 218f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_REL_OFFSET esi, 0 219f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl %edx 220f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 221f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_REL_OFFSET edx, 0 222f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl %ecx 223f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 224f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_REL_OFFSET ecx, 0 225f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl %ebx 226f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 227f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_REL_OFFSET ebx, 0 228f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl $(__USER_DS), %edx 229f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl %edx, %ds 230f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl %edx, %es 231f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl $(__KERNEL_PERCPU), %edx 232464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge movl %edx, %fs 233ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo SET_KERNEL_GS %edx 234f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 2351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 236f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro RESTORE_INT_REGS 237f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo popl %ebx 238f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -4 239f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_RESTORE ebx 240f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo popl %ecx 241f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -4 242f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_RESTORE ecx 243f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo popl %edx 244f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -4 245f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_RESTORE edx 246f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo popl %esi 247f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -4 248f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_RESTORE esi 249f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo popl %edi 250f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -4 251f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_RESTORE edi 252f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo popl %ebp 253f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -4 254f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_RESTORE ebp 255f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo popl %eax 256f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -4 257fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_RESTORE eax 258f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 2591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 260ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo.macro RESTORE_REGS pop=0 261f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo RESTORE_INT_REGS 262f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo1: popl %ds 263f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -4 264f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_RESTORE ds;*/ 265f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo2: popl %es 266f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -4 267f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_RESTORE es;*/ 268f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo3: popl %fs 269f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -4 270f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_RESTORE fs;*/ 271ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo POP_GS \pop 272f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.pushsection .fixup, "ax" 273f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo4: movl $0, (%esp) 274f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo jmp 1b 275f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo5: movl $0, (%esp) 276f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo jmp 2b 277f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo6: movl $0, (%esp) 278f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo jmp 3b 279f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.section __ex_table, "a" 280f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo .align 4 281f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo .long 1b, 4b 282f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo .long 2b, 5b 283f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo .long 3b, 6b 284f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge.popsection 285ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo POP_GS_EX 286f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 2871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 288f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro RING0_INT_FRAME 289f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_STARTPROC simple 290f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_SIGNAL_FRAME 291f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_DEF_CFA esp, 3*4 292f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_OFFSET cs, -2*4;*/ 293fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_OFFSET eip, -3*4 294f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 295fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich 296f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro RING0_EC_FRAME 297f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_STARTPROC simple 298f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_SIGNAL_FRAME 299f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_DEF_CFA esp, 4*4 300f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_OFFSET cs, -2*4;*/ 301fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_OFFSET eip, -3*4 302f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 303fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich 304f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro RING0_PTREGS_FRAME 305f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_STARTPROC simple 306f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_SIGNAL_FRAME 307f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_DEF_CFA esp, PT_OLDESP-PT_EBX 308f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/ 309f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_OFFSET eip, PT_EIP-PT_OLDESP 310f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/ 311f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/ 312f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_OFFSET eax, PT_EAX-PT_OLDESP 313f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_OFFSET ebp, PT_EBP-PT_OLDESP 314f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_OFFSET edi, PT_EDI-PT_OLDESP 315f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_OFFSET esi, PT_ESI-PT_OLDESP 316f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_OFFSET edx, PT_EDX-PT_OLDESP 317f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_OFFSET ecx, PT_ECX-PT_OLDESP 318eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge CFI_OFFSET ebx, PT_EBX-PT_OLDESP 319f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 3201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(ret_from_fork) 322fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_STARTPROC 3231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax 32425d7dfdaf3404bb31b8f55283fd2c456cb7b4001Markus Armbruster CFI_ADJUST_CFA_OFFSET 4 3251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call schedule_tail 3261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 3271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %eax 328fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 32947a5c6fa0e204a2b63309c648bb2fde36836c826Linus Torvalds pushl $0x0202 # Reset kernel eflags 33047a5c6fa0e204a2b63309c648bb2fde36836c826Linus Torvalds CFI_ADJUST_CFA_OFFSET 4 33147a5c6fa0e204a2b63309c648bb2fde36836c826Linus Torvalds popfl 33247a5c6fa0e204a2b63309c648bb2fde36836c826Linus Torvalds CFI_ADJUST_CFA_OFFSET -4 3331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp syscall_exit 334fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 33547a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(ret_from_fork) 3361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 3381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Return to user mode is not as complex as all this looks, 3391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * but we want the default path for a system call return to 3401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * go as quickly as possible which is why some of this is 3411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * less clear than it otherwise should be. 3421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 3431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # userspace resumption stub bypassing syscall exit tracing 3451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 346fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_PTREGS_FRAME 3471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsret_from_exception: 348139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell preempt_stop(CLBR_ANY) 3491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsret_from_intr: 3501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 3514031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelovcheck_userspace: 352eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS 353eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movb PT_CS(%esp), %al 354ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax 35578be3706b21a232310590fe00258b224177ac05fRusty Russell cmpl $USER_RPL, %eax 35678be3706b21a232310590fe00258b224177ac05fRusty Russell jb resume_kernel # not returning to v8086 or userspace 357f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge 3581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(resume_userspace) 359c7e872e7da5514d014707a407ea562d197cc0136Peter Zijlstra LOCKDEP_SYS_EXIT 360139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 3611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # setting need_resched or sigpending 3621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # between sampling and the iret 363e32e58a96de4ac35a03349db2ab69f263ded958fPeter Zijlstra TRACE_IRQS_OFF 3641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx 3651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds andl $_TIF_WORK_MASK, %ecx # is there any work to be done on 3661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # int/exception return? 3671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne work_pending 3681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp restore_all 36947a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(ret_from_exception) 3701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef CONFIG_PREEMPT 3721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(resume_kernel) 373139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_ANY) 3741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ? 3751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz restore_nocheck 3761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsneed_resched: 3771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx # need_resched set ? 3781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testb $_TIF_NEED_RESCHED, %cl 3791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz restore_all 380ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off (exception path) ? 3811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz restore_all 3821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call preempt_schedule_irq 3831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp need_resched 38447a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(resume_kernel) 3851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 386fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 3871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* SYSENTER_RETURN points to after the "sysenter" instruction in 3891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */ 3901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # sysenter call handler stub 3920aa97fb22624f18e5925d702ab0364d3838cfd91Roland McGrathENTRY(ia32_sysenter_target) 393fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_STARTPROC simple 394adf1423698f00d00b267f7dca8231340ce7d65efJan Beulich CFI_SIGNAL_FRAME 395fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_DEF_CFA esp, 0 396fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REGISTER esp, ebp 397faca62273b602ab482fb7d3d940dbf41ef08b00eH. Peter Anvin movl TSS_sysenter_sp0(%esp),%esp 3981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssysenter_past_esp: 39955f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar /* 400d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge * Interrupts are disabled here, but we can't trace it until 401d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge * enough kernel state to call TRACE_IRQS_OFF can be called - but 402d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge * we immediately enable interrupts at that point anyway. 40355f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar */ 4041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $(__USER_DS) 405fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 406fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_REL_OFFSET ss, 0*/ 4071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ebp 408fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 409fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET esp, 0 4101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushfl 411d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge orl $X86_EFLAGS_IF, (%esp) 412fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 4131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $(__USER_CS) 414fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 415fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_REL_OFFSET cs, 0*/ 416e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar /* 417e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar * Push current_thread_info()->sysenter_return to the stack. 418e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar * A tiny bit of offset fixup is necessary - 4*4 means the 4 words 419e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar * pushed above; +8 corresponds to copy_thread's esp0 setting. 420e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar */ 421e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp) 422fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 423fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET eip, 0 4241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 425d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge pushl %eax 426d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET 4 427d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge SAVE_ALL 428d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge ENABLE_INTERRUPTS(CLBR_NONE) 429d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge 4301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 4311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Load the potential sixth argument from user stack. 4321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Careful about security. 4331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 4341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $__PAGE_OFFSET-3,%ebp 4351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jae syscall_fault 4361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds1: movl (%ebp),%ebp 437d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge movl %ebp,PT_EBP(%esp) 4381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.section __ex_table,"a" 4391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .align 4 4401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .long 1b,syscall_fault 4411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.previous 4421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 4441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 44588200bc28da38bcda1cb1bd218216e83b426d8a8Jaswinder Singh Rajput testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp) 446af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jnz sysenter_audit 447af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrathsysenter_do_call: 4481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $(nr_syscalls), %eax 4491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jae syscall_badsys 4501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call *sys_call_table(,%eax,4) 451eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl %eax,PT_EAX(%esp) 452c7e872e7da5514d014707a407ea562d197cc0136Peter Zijlstra LOCKDEP_SYS_EXIT 45342c24fa22e86365055fc931d833f26165e687c19Jeremy Fitzhardinge DISABLE_INTERRUPTS(CLBR_ANY) 45455f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 4551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx 45688200bc28da38bcda1cb1bd218216e83b426d8a8Jaswinder Singh Rajput testl $_TIF_ALLWORK_MASK, %ecx 457af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jne sysexit_audit 458af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrathsysenter_exit: 4591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* if something modifies registers it must also disable sysexit */ 460eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_EIP(%esp), %edx 461eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_OLDESP(%esp), %ecx 4621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %ebp,%ebp 46355f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_ON 464464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge1: mov PT_FS(%esp), %fs 465ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo PTGS_TO_GS 466d75cd22fdd5f7d203fb60014d426942df33dd9a6Jeremy Fitzhardinge ENABLE_INTERRUPTS_SYSEXIT 467af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath 468af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#ifdef CONFIG_AUDITSYSCALL 469af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrathsysenter_audit: 47088200bc28da38bcda1cb1bd218216e83b426d8a8Jaswinder Singh Rajput testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%ebp) 471af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jnz syscall_trace_entry 472af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath addl $4,%esp 473af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath CFI_ADJUST_CFA_OFFSET -4 474af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath /* %esi already in 8(%esp) 6th arg: 4th syscall arg */ 475af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath /* %edx already in 4(%esp) 5th arg: 3rd syscall arg */ 476af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath /* %ecx already in 0(%esp) 4th arg: 2nd syscall arg */ 477af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl %ebx,%ecx /* 3rd arg: 1st syscall arg */ 478af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl %eax,%edx /* 2nd arg: syscall number */ 479af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl $AUDIT_ARCH_I386,%eax /* 1st arg: audit arch */ 480af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath call audit_syscall_entry 481af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath pushl %ebx 482af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath CFI_ADJUST_CFA_OFFSET 4 483af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl PT_EAX(%esp),%eax /* reload syscall number */ 484af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jmp sysenter_do_call 485af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath 486af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrathsysexit_audit: 48788200bc28da38bcda1cb1bd218216e83b426d8a8Jaswinder Singh Rajput testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx 488af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jne syscall_exit_work 489af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath TRACE_IRQS_ON 490af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath ENABLE_INTERRUPTS(CLBR_ANY) 491af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl %eax,%edx /* second arg, syscall return value */ 492af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath cmpl $0,%eax /* is it < 0? */ 493af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath setl %al /* 1 if so, 0 if not */ 494af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movzbl %al,%eax /* zero-extend that */ 495af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath inc %eax /* first arg, 0->1(AUDITSC_SUCCESS), 1->2(AUDITSC_FAILURE) */ 496af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath call audit_syscall_exit 497af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath DISABLE_INTERRUPTS(CLBR_ANY) 498af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath TRACE_IRQS_OFF 499af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl TI_flags(%ebp), %ecx 50088200bc28da38bcda1cb1bd218216e83b426d8a8Jaswinder Singh Rajput testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx 501af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jne syscall_exit_work 502af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl PT_EAX(%esp),%eax /* reload syscall return value */ 503af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jmp sysenter_exit 504af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#endif 505af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath 506fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 507f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge.pushsection .fixup,"ax" 508464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge2: movl $0,PT_FS(%esp) 509f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge jmp 1b 510f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge.section __ex_table,"a" 511f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge .align 4 512f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge .long 1b,2b 513f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge.popsection 514ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo PTGS_TO_GS_EX 5150aa97fb22624f18e5925d702ab0364d3838cfd91Roland McGrathENDPROC(ia32_sysenter_target) 5161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # system call handler stub 5181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(system_call) 519fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME # can't unwind into user space anyway 5201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax # save orig_eax 521fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 5221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds SAVE_ALL 5231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 524ed75e8d58010fdc06e2c3a81bfbebae92314c7e3Laurent Vivier # system call tracing in operation / emulation 52588200bc28da38bcda1cb1bd218216e83b426d8a8Jaswinder Singh Rajput testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp) 5261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz syscall_trace_entry 5271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $(nr_syscalls), %eax 5281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jae syscall_badsys 5291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_call: 5301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call *sys_call_table(,%eax,4) 531eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl %eax,PT_EAX(%esp) # store the return value 5321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_exit: 533c7e872e7da5514d014707a407ea562d197cc0136Peter Zijlstra LOCKDEP_SYS_EXIT 534139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 5351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # setting need_resched or sigpending 5361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # between sampling and the iret 53755f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 5381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx 53988200bc28da38bcda1cb1bd218216e83b426d8a8Jaswinder Singh Rajput testl $_TIF_ALLWORK_MASK, %ecx # current->work 5401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne syscall_exit_work 5411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsrestore_all: 543eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS 544eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge # Warning: PT_OLDSS(%esp) contains the wrong/random values if we 5455df240826c90afdc7956f55a004ea6b702df9203Stas Sergeev # are returning to the kernel. 5465df240826c90afdc7956f55a004ea6b702df9203Stas Sergeev # See comments in process.c:copy_thread() for details. 547eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movb PT_OLDSS(%esp), %ah 548eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movb PT_CS(%esp), %al 549ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax 55078be3706b21a232310590fe00258b224177ac05fRusty Russell cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax 551fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REMEMBER_STATE 5521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds je ldt_ss # returning to user-space with LDT SS 5531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsrestore_nocheck: 55455f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_IRET 55555f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnarrestore_nocheck_notrace: 556ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo RESTORE_REGS 4 # skip orig_eax/error_code 557fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 558f7f3d791e61d7baf8b0aee0384fdd469c0d2ac9bAdrian Bunkirq_return: 5593701d863b43d05ffeb223d269583398f914fb5d3Ingo Molnar INTERRUPT_RETURN 5601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.section .fixup,"ax" 56190e9f53662826db3cdd6d99bd394d727b05160c1Jeremy FitzhardingeENTRY(iret_exc) 562a879cbbb34cbecfa9707fbb6e5a00c503ac1ecb9Linus Torvalds pushl $0 # no error code 563a879cbbb34cbecfa9707fbb6e5a00c503ac1ecb9Linus Torvalds pushl $do_iret_error 564a879cbbb34cbecfa9707fbb6e5a00c503ac1ecb9Linus Torvalds jmp error_code 5651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.previous 5661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.section __ex_table,"a" 5671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .align 4 5683701d863b43d05ffeb223d269583398f914fb5d3Ingo Molnar .long irq_return,iret_exc 5691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.previous 5701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 571fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_RESTORE_STATE 5721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsldt_ss: 573eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge larl PT_OLDSS(%esp), %eax 5741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz restore_nocheck 5751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testl $0x00400000, %eax # returning to 32bit stack? 5761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz restore_nocheck # allright, normal return 577d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell 578d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell#ifdef CONFIG_PARAVIRT 579d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell /* 580d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * The kernel can't run on a non-flat stack if paravirt mode 581d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * is active. Rather than try to fixup the high bits of 582d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * ESP, bypass this code entirely. This may break DOSemu 583d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * and/or Wine support in a paravirt VM, although the option 584d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * is still available to implement the setting of the high 585d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * 16-bits in the INTERRUPT_RETURN paravirt-op. 586d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell */ 58793b1eab3d29e7ea32ee583de3362da84db06ded8Jeremy Fitzhardinge cmpl $0, pv_info+PARAVIRT_enabled 588d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell jne restore_nocheck 589d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell#endif 590d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell 5911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* If returning to userspace with 16bit stack, 5921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * try to fix the higher word of ESP, as the CPU 5931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * won't restore it. 5941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This is an "official" bug of all the x86-compatible 5951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * CPUs, which we can try to work around to make 5961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * dosemu and wine happy. */ 597eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_OLDESP(%esp), %eax 598be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev movl %esp, %edx 599be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev call patch_espfix_desc 600be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev pushl $__ESPFIX_SS 601be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev CFI_ADJUST_CFA_OFFSET 4 602be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev pushl %eax 603be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev CFI_ADJUST_CFA_OFFSET 4 604139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_EAX) 60555f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 606be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev lss (%esp), %esp 607be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev CFI_ADJUST_CFA_OFFSET -8 608be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev jmp restore_nocheck 609fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 61047a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichENDPROC(system_call) 6111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # perform work that needs to be done immediately before resumption 6131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 614fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_PTREGS_FRAME # can't unwind into user space anyway 6151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldswork_pending: 6161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testb $_TIF_NEED_RESCHED, %cl 6171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz work_notifysig 6181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldswork_resched: 6191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call schedule 620c7e872e7da5514d014707a407ea562d197cc0136Peter Zijlstra LOCKDEP_SYS_EXIT 621139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 6221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # setting need_resched or sigpending 6231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # between sampling and the iret 62455f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 6251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx 6261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds andl $_TIF_WORK_MASK, %ecx # is there any work to be done other 6271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # than syscall tracing? 6281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz restore_all 6291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testb $_TIF_NEED_RESCHED, %cl 6301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz work_resched 6311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldswork_notifysig: # deal with pending signals and 6331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # notify-resume requests 63474b47a7844501445d41d704fe7c626f4b1819508Joe Korty#ifdef CONFIG_VM86 635ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov testl $X86_EFLAGS_VM, PT_EFLAGS(%esp) 6361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp, %eax 6371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne work_notifysig_v86 # returning to kernel-space or 6381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # vm86-space 6391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %edx, %edx 6401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_notify_resume 6414031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov jmp resume_userspace_sig 6421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 6441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldswork_notifysig_v86: 6451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ecx # save ti_flags for do_notify_resume 646fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 6471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call save_v86_state # %eax contains pt_regs pointer 6481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %ecx 649fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 6501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %eax, %esp 65174b47a7844501445d41d704fe7c626f4b1819508Joe Korty#else 65274b47a7844501445d41d704fe7c626f4b1819508Joe Korty movl %esp, %eax 65374b47a7844501445d41d704fe7c626f4b1819508Joe Korty#endif 6541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %edx, %edx 6551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_notify_resume 6564031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov jmp resume_userspace_sig 65747a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(work_pending) 6581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # perform syscall exit tracing 6601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 6611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_trace_entry: 662eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl $-ENOSYS,PT_EAX(%esp) 6631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp, %eax 664d4d67150165df8bf1cc05e532f6efca96f907cabRoland McGrath call syscall_trace_enter 665d4d67150165df8bf1cc05e532f6efca96f907cabRoland McGrath /* What it returned is what we'll actually use. */ 6661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $(nr_syscalls), %eax 6671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnae syscall_call 6681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp syscall_exit 66947a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(syscall_trace_entry) 6701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # perform syscall exit tracing 6721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 6731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_exit_work: 67488200bc28da38bcda1cb1bd218216e83b426d8a8Jaswinder Singh Rajput testl $_TIF_WORK_SYSCALL_EXIT, %ecx 6751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz work_pending 67655f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_ON 677d4d67150165df8bf1cc05e532f6efca96f907cabRoland McGrath ENABLE_INTERRUPTS(CLBR_ANY) # could let syscall_trace_leave() call 6781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # schedule() instead 6791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp, %eax 680d4d67150165df8bf1cc05e532f6efca96f907cabRoland McGrath call syscall_trace_leave 6811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp resume_userspace 68247a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(syscall_exit_work) 683fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 6841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 685fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME # can't unwind into user space anyway 6861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_fault: 6871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 688eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl $-EFAULT,PT_EAX(%esp) 6891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp resume_userspace 69047a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(syscall_fault) 6911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_badsys: 693eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl $-ENOSYS,PT_EAX(%esp) 6941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp resume_userspace 69547a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(syscall_badsys) 696fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 6971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 698253f29a4ae9cc6cdc7b94f96517f27a93885a6ceBrian Gerst/* 699253f29a4ae9cc6cdc7b94f96517f27a93885a6ceBrian Gerst * System calls that need a pt_regs pointer. 700253f29a4ae9cc6cdc7b94f96517f27a93885a6ceBrian Gerst */ 701253f29a4ae9cc6cdc7b94f96517f27a93885a6ceBrian Gerst#define PTREGSCALL(name) \ 702253f29a4ae9cc6cdc7b94f96517f27a93885a6ceBrian Gerst ALIGN; \ 703253f29a4ae9cc6cdc7b94f96517f27a93885a6ceBrian Gerstptregs_##name: \ 704253f29a4ae9cc6cdc7b94f96517f27a93885a6ceBrian Gerst leal 4(%esp),%eax; \ 705253f29a4ae9cc6cdc7b94f96517f27a93885a6ceBrian Gerst jmp sys_##name; 706253f29a4ae9cc6cdc7b94f96517f27a93885a6ceBrian Gerst 707253f29a4ae9cc6cdc7b94f96517f27a93885a6ceBrian GerstPTREGSCALL(iopl) 708253f29a4ae9cc6cdc7b94f96517f27a93885a6ceBrian GerstPTREGSCALL(fork) 709253f29a4ae9cc6cdc7b94f96517f27a93885a6ceBrian GerstPTREGSCALL(clone) 710253f29a4ae9cc6cdc7b94f96517f27a93885a6ceBrian GerstPTREGSCALL(vfork) 711253f29a4ae9cc6cdc7b94f96517f27a93885a6ceBrian GerstPTREGSCALL(execve) 712253f29a4ae9cc6cdc7b94f96517f27a93885a6ceBrian GerstPTREGSCALL(sigaltstack) 713253f29a4ae9cc6cdc7b94f96517f27a93885a6ceBrian GerstPTREGSCALL(sigreturn) 714253f29a4ae9cc6cdc7b94f96517f27a93885a6ceBrian GerstPTREGSCALL(rt_sigreturn) 715253f29a4ae9cc6cdc7b94f96517f27a93885a6ceBrian GerstPTREGSCALL(vm86) 716253f29a4ae9cc6cdc7b94f96517f27a93885a6ceBrian GerstPTREGSCALL(vm86old) 717253f29a4ae9cc6cdc7b94f96517f27a93885a6ceBrian Gerst 718f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro FIXUP_ESPFIX_STACK 719f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /* since we are on a wrong stack, we cant make it a C code :( */ 720f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo PER_CPU(gdt_page, %ebx) 721f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah) 722f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo addl %esp, %eax 723f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl $__KERNEL_DS 724f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 725f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl %eax 726f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 727f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo lss (%esp), %esp 728f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -8 729f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 730f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro UNWIND_ESPFIX_STACK 731f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl %ss, %eax 732f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /* see if on espfix stack */ 733f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo cmpw $__ESPFIX_SS, %ax 734f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo jne 27f 735f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl $__KERNEL_DS, %eax 736f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl %eax, %ds 737f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl %eax, %es 738f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /* switch to normal stack */ 739f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo FIXUP_ESPFIX_STACK 740f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo27: 741f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 7421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 744b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin * Build the entry stubs and pointer table with some assembler magic. 745b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin * We pack 7 stubs into a single 32-byte chunk, which will fit in a 746b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin * single cache line on all modern x86 implementations. 7471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 7484687518c4cb7807fbeff21770e309080f9eb7f2fH. Peter Anvin.section .init.rodata,"a" 7491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(interrupt) 7501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.text 751b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .p2align 5 752b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .p2align CONFIG_X86_L1_CACHE_SHIFT 7531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(irq_entries_start) 754fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 7554687518c4cb7807fbeff21770e309080f9eb7f2fH. Peter Anvinvector=FIRST_EXTERNAL_VECTOR 756b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin.rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7 757b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .balign 32 758b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .rept 7 759b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .if vector < NR_VECTORS 7608665596ec05498525014436520b316ba174a068aH. Peter Anvin .if vector <> FIRST_EXTERNAL_VECTOR 761fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 762b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .endif 763b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin1: pushl $(~vector+0x80) /* Note: always in signed byte range */ 764fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7658665596ec05498525014436520b316ba174a068aH. Peter Anvin .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6 766b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin jmp 2f 767b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .endif 768b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .previous 7691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .long 1b 770b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .text 7711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvector=vector+1 772b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .endif 773b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .endr 774b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin2: jmp common_interrupt 7751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.endr 77647a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(irq_entries_start) 77747a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich 77847a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich.previous 77947a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(interrupt) 78047a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich.previous 7811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 78255f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar/* 78355f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar * the CPU automatically disables interrupts when executing an IRQ vector, 78455f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar * so IRQ-flags tracing has to follow that: 78555f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar */ 786b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .p2align CONFIG_X86_L1_CACHE_SHIFT 7871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldscommon_interrupt: 788b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin addl $-0x80,(%esp) /* Adjust vector into the [-256,-1] range */ 7891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds SAVE_ALL 79055f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 7911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp,%eax 7921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_IRQ 7931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp ret_from_intr 79447a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichENDPROC(common_interrupt) 795fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 7961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 79702cf94c370e0dc9bf408fe45eb86fe9ad58eaf7fTejun Heo#define BUILD_INTERRUPT3(name, nr, fn) \ 7981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(name) \ 799fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME; \ 80019eadf98c8167eac843580683317b99572e2abf0Rusty Russell pushl $~(nr); \ 801fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4; \ 802fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich SAVE_ALL; \ 80355f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF \ 8041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp,%eax; \ 80502cf94c370e0dc9bf408fe45eb86fe9ad58eaf7fTejun Heo call fn; \ 80655f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar jmp ret_from_intr; \ 80747a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich CFI_ENDPROC; \ 80847a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichENDPROC(name) 8091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 81002cf94c370e0dc9bf408fe45eb86fe9ad58eaf7fTejun Heo#define BUILD_INTERRUPT(name, nr) BUILD_INTERRUPT3(name, nr, smp_##name) 81102cf94c370e0dc9bf408fe45eb86fe9ad58eaf7fTejun Heo 8121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* The include is where all of the SMP etc. interrupts come from */ 8131164dd0099c0d79146a55319670f57ab7ad1d352Ingo Molnar#include <asm/entry_arch.h> 8141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(coprocessor_error) 816fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 8171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 818fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_coprocessor_error 820fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 822fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 82347a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(coprocessor_error) 8241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(simd_coprocessor_error) 826fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 8271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 828fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_simd_coprocessor_error 830fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 832fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 83347a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(simd_coprocessor_error) 8341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(device_not_available) 836fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 8371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $-1 # mark this as an int 838fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8397643e9b936b4af31ba4851eb7d5b3a3bfad52502Alexander van Heukelum pushl $do_device_not_available 840fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8417643e9b936b4af31ba4851eb7d5b3a3bfad52502Alexander van Heukelum jmp error_code 842fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 84347a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(device_not_available) 8441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 845d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell#ifdef CONFIG_PARAVIRT 846d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty RussellENTRY(native_iret) 8473701d863b43d05ffeb223d269583398f914fb5d3Ingo Molnar iret 848d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell.section __ex_table,"a" 849d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell .align 4 8503701d863b43d05ffeb223d269583398f914fb5d3Ingo Molnar .long native_iret, iret_exc 851d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell.previous 85247a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(native_iret) 853d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell 854d75cd22fdd5f7d203fb60014d426942df33dd9a6Jeremy FitzhardingeENTRY(native_irq_enable_sysexit) 855d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell sti 856d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell sysexit 857d75cd22fdd5f7d203fb60014d426942df33dd9a6Jeremy FitzhardingeEND(native_irq_enable_sysexit) 858d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell#endif 859d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell 8601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(overflow) 861fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 8621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 863fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_overflow 865fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 867fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 86847a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(overflow) 8691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(bounds) 871fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 8721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 873fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_bounds 875fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 877fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 87847a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(bounds) 8791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(invalid_op) 881fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 8821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 883fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_invalid_op 885fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 887fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 88847a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(invalid_op) 8891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(coprocessor_segment_overrun) 891fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 8921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 893fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_coprocessor_segment_overrun 895fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 897fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 89847a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(coprocessor_segment_overrun) 8991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(invalid_TSS) 901fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_EC_FRAME 9021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_invalid_TSS 903fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 905fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 90647a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(invalid_TSS) 9071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(segment_not_present) 909fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_EC_FRAME 9101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_segment_not_present 911fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 913fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 91447a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(segment_not_present) 9151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(stack_segment) 917fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_EC_FRAME 9181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_stack_segment 919fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 921fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 92247a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(stack_segment) 9231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(alignment_check) 925fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_EC_FRAME 9261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_alignment_check 927fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 929fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 93047a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(alignment_check) 9311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 932d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.PENTRY(divide_error) 933d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P RING0_INT_FRAME 934d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P pushl $0 # no error code 935d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P CFI_ADJUST_CFA_OFFSET 4 936d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P pushl $do_divide_error 937fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 939fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 94047a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(divide_error) 9411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef CONFIG_X86_MCE 9431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(machine_check) 944fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 9451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 946fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 947d2f6f7aeee890df445be29a60e34925ec15f620cAndi Kleen pushl machine_check_vector 948fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 950fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 95147a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(machine_check) 9521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 9531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(spurious_interrupt_bug) 955fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 9561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 957fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_spurious_interrupt_bug 959fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 961fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 96247a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(spurious_interrupt_bug) 9631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 96402ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi KleenENTRY(kernel_thread_helper) 96502ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen pushl $0 # fake return address for unwinder 96602ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen CFI_STARTPROC 96702ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen movl %edx,%eax 96802ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen push %edx 96902ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen CFI_ADJUST_CFA_OFFSET 4 97002ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen call *%ebx 97102ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen push %eax 97202ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen CFI_ADJUST_CFA_OFFSET 4 97302ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen call do_exit 9745f5db591326779a80cfe490c5d6b6ce9fac08b31jia zhang ud2 # padding for call trace 97502ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen CFI_ENDPROC 97602ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi KleenENDPROC(kernel_thread_helper) 97702ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen 9785ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge#ifdef CONFIG_XEN 979e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge/* Xen doesn't set %esp to be precisely what the normal sysenter 980e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge entrypoint expects, so fix it up before using the normal path. */ 981e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy FitzhardingeENTRY(xen_sysenter_target) 982e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge RING0_INT_FRAME 983e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge addl $5*4, %esp /* remove xen-provided frame */ 9842ddf9b7b3e6660199269e34cfa27148440ddc3bfJan Beulich CFI_ADJUST_CFA_OFFSET -5*4 985e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge jmp sysenter_past_esp 986557d7d4e294ee6fb1db0cb6c1ec97a1c908b880dGlauber Costa CFI_ENDPROC 987e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge 9885ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy FitzhardingeENTRY(xen_hypervisor_callback) 9895ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_STARTPROC 9905ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge pushl $0 9915ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET 4 9925ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge SAVE_ALL 9935ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge TRACE_IRQS_OFF 9949ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge 9959ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge /* Check to see if we got the event in the critical 9969ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge region in xen_iret_direct, after we've reenabled 9979ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge events and checked for pending events. This simulates 9989ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge iret instruction's behaviour where it delivers a 9999ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge pending interrupt when enabling interrupts. */ 10009ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge movl PT_EIP(%esp),%eax 10019ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge cmpl $xen_iret_start_crit,%eax 10029ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge jb 1f 10039ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge cmpl $xen_iret_end_crit,%eax 10049ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge jae 1f 10059ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge 10060f2c87695219b1129ccf93e0f58acdcdd49724b9Jeremy Fitzhardinge jmp xen_iret_crit_fixup 1007e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge 1008e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy FitzhardingeENTRY(xen_do_upcall) 1009b77797fb2bf31bf076e6b69736119bc6a077525bJeremy Fitzhardinge1: mov %esp, %eax 10105ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge call xen_evtchn_do_upcall 10115ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp ret_from_intr 10125ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ENDPROC 10135ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy FitzhardingeENDPROC(xen_hypervisor_callback) 10145ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge 10155ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# Hypervisor uses this for application faults while it executes. 10165ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# We get here for two reasons: 10175ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# 1. Fault while reloading DS, ES, FS or GS 10185ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# 2. Fault while executing IRET 10195ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# Category 1 we fix up by reattempting the load, and zeroing the segment 10205ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# register if the load fails. 10215ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# Category 2 we fix up by jumping to do_iret_error. We cannot use the 10225ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# normal Linux return path in this case because if we use the IRET hypercall 10235ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# to pop the stack frame we end up in an infinite loop of failsafe callbacks. 10245ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# We distinguish between categories by maintaining a status value in EAX. 10255ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy FitzhardingeENTRY(xen_failsafe_callback) 10265ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_STARTPROC 10275ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge pushl %eax 10285ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET 4 10295ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl $1,%eax 10305ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge1: mov 4(%esp),%ds 10315ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge2: mov 8(%esp),%es 10325ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge3: mov 12(%esp),%fs 10335ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge4: mov 16(%esp),%gs 10345ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge testl %eax,%eax 10355ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge popl %eax 10365ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET -4 10375ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge lea 16(%esp),%esp 10385ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET -16 10395ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jz 5f 10405ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge addl $16,%esp 10415ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp iret_exc # EAX != 0 => Category 2 (Bad IRET) 10425ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge5: pushl $0 # EAX == 0 => Category 1 (Bad segment) 10435ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET 4 10445ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge SAVE_ALL 10455ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp ret_from_exception 10465ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ENDPROC 10475ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge 10485ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge.section .fixup,"ax" 10495ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge6: xorl %eax,%eax 10505ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl %eax,4(%esp) 10515ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp 1b 10525ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge7: xorl %eax,%eax 10535ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl %eax,8(%esp) 10545ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp 2b 10555ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge8: xorl %eax,%eax 10565ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl %eax,12(%esp) 10575ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp 3b 10585ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge9: xorl %eax,%eax 10595ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl %eax,16(%esp) 10605ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp 4b 10615ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge.previous 10625ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge.section __ex_table,"a" 10635ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge .align 4 10645ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge .long 1b,6b 10655ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge .long 2b,7b 10665ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge .long 3b,8b 10675ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge .long 4b,9b 10685ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge.previous 10695ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy FitzhardingeENDPROC(xen_failsafe_callback) 10705ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge 10715ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge#endif /* CONFIG_XEN */ 10725ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge 1073606576ce816603d9fe1fb453a88bc6eea16ca709Steven Rostedt#ifdef CONFIG_FUNCTION_TRACER 1074d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt#ifdef CONFIG_DYNAMIC_FTRACE 1075d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 1076d61f82d06672f57fca410da6f7fffd15867db622Steven RostedtENTRY(mcount) 1077d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt ret 1078d61f82d06672f57fca410da6f7fffd15867db622Steven RostedtEND(mcount) 1079d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 1080d61f82d06672f57fca410da6f7fffd15867db622Steven RostedtENTRY(ftrace_caller) 108160a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt cmpl $0, function_trace_stop 108260a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt jne ftrace_stub 108360a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt 1084d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt pushl %eax 1085d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt pushl %ecx 1086d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt pushl %edx 1087d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt movl 0xc(%esp), %eax 1088d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt movl 0x4(%ebp), %edx 1089395a59d0f8e86bb39cd700c3d185d30c670bb958Abhishek Sagar subl $MCOUNT_INSN_SIZE, %eax 1090d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 1091d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt.globl ftrace_call 1092d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedtftrace_call: 1093d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt call ftrace_stub 1094d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 1095d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt popl %edx 1096d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt popl %ecx 1097d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt popl %eax 10985a45cfe1c64862e8cd3b0d79d7c4ba71c3118915Steven Rostedt#ifdef CONFIG_FUNCTION_GRAPH_TRACER 10995a45cfe1c64862e8cd3b0d79d7c4ba71c3118915Steven Rostedt.globl ftrace_graph_call 11005a45cfe1c64862e8cd3b0d79d7c4ba71c3118915Steven Rostedtftrace_graph_call: 11015a45cfe1c64862e8cd3b0d79d7c4ba71c3118915Steven Rostedt jmp ftrace_stub 11025a45cfe1c64862e8cd3b0d79d7c4ba71c3118915Steven Rostedt#endif 1103d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 1104d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt.globl ftrace_stub 1105d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedtftrace_stub: 1106d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt ret 1107d61f82d06672f57fca410da6f7fffd15867db622Steven RostedtEND(ftrace_caller) 1108d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 1109d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt#else /* ! CONFIG_DYNAMIC_FTRACE */ 1110d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 111116444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de MeloENTRY(mcount) 111260a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt cmpl $0, function_trace_stop 111360a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt jne ftrace_stub 111460a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt 111516444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo cmpl $ftrace_stub, ftrace_trace_function 111616444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo jnz trace 1117fb52607afcd0629776f1dc9e657647ceae81dd50Frederic Weisbecker#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1118c2324b694fa8ffee382a124198c68754088e483cIngo Molnar cmpl $ftrace_stub, ftrace_graph_return 1119fb52607afcd0629776f1dc9e657647ceae81dd50Frederic Weisbecker jnz ftrace_graph_caller 1120e49dc19c6a19ea112fcb94b7c62ec62cdd5c08aaSteven Rostedt 1121e49dc19c6a19ea112fcb94b7c62ec62cdd5c08aaSteven Rostedt cmpl $ftrace_graph_entry_stub, ftrace_graph_entry 1122e49dc19c6a19ea112fcb94b7c62ec62cdd5c08aaSteven Rostedt jnz ftrace_graph_caller 1123caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker#endif 112416444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo.globl ftrace_stub 112516444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Meloftrace_stub: 112616444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo ret 112716444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo 112816444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo /* taken from glibc */ 112916444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melotrace: 113016444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo pushl %eax 113116444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo pushl %ecx 113216444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo pushl %edx 113316444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo movl 0xc(%esp), %eax 113416444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo movl 0x4(%ebp), %edx 1135395a59d0f8e86bb39cd700c3d185d30c670bb958Abhishek Sagar subl $MCOUNT_INSN_SIZE, %eax 113616444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo 1137d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt call *ftrace_trace_function 113816444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo 113916444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo popl %edx 114016444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo popl %ecx 114116444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo popl %eax 114216444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo jmp ftrace_stub 114316444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de MeloEND(mcount) 1144d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt#endif /* CONFIG_DYNAMIC_FTRACE */ 1145606576ce816603d9fe1fb453a88bc6eea16ca709Steven Rostedt#endif /* CONFIG_FUNCTION_TRACER */ 114616444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo 1147fb52607afcd0629776f1dc9e657647ceae81dd50Frederic Weisbecker#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1148fb52607afcd0629776f1dc9e657647ceae81dd50Frederic WeisbeckerENTRY(ftrace_graph_caller) 1149e7d3737ea1b102030f44e96c97754101e41515f0Frederic Weisbecker cmpl $0, function_trace_stop 1150e7d3737ea1b102030f44e96c97754101e41515f0Frederic Weisbecker jne ftrace_stub 1151e7d3737ea1b102030f44e96c97754101e41515f0Frederic Weisbecker 1152caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %eax 1153caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %ecx 1154caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %edx 11551dc1c6adf38bc5799d1594681645ced40ced4b6bFrederic Weisbecker movl 0xc(%esp), %edx 1156caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker lea 0x4(%ebp), %eax 1157bb4304c71c97bf727ec43cd2f195c2c237c27fd3Steven Rostedt subl $MCOUNT_INSN_SIZE, %edx 1158caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker call prepare_ftrace_return 1159caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %edx 1160caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %ecx 1161caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %eax 1162e7d3737ea1b102030f44e96c97754101e41515f0Frederic Weisbecker ret 1163fb52607afcd0629776f1dc9e657647ceae81dd50Frederic WeisbeckerEND(ftrace_graph_caller) 1164caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker 1165caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker.globl return_to_handler 1166caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbeckerreturn_to_handler: 1167caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl $0 1168caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %eax 1169caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %ecx 1170caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %edx 1171caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker call ftrace_return_to_handler 1172caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker movl %eax, 0xc(%esp) 1173caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %edx 1174caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %ecx 1175caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %eax 1176caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker ret 1177e7d3737ea1b102030f44e96c97754101e41515f0Frederic Weisbecker#endif 117816444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo 1179bb152f53120d66c98c1f16518407df6a84f23714Arjan van de Ven.section .rodata,"a" 1180541054d935a1ec89916977cbf0d16ddb71b3ff5cThomas Gleixner#include "syscall_table_32.S" 11811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 11821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_table_size=(.-sys_call_table) 1183d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1184d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum/* 1185d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * Some functions should be protected against kprobes 1186d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum */ 1187d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum .pushsection .kprobes.text, "ax" 1188d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1189d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumENTRY(page_fault) 1190d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum RING0_EC_FRAME 1191d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl $do_page_fault 1192d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1193d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum ALIGN 1194d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelumerror_code: 1195ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo /* the function address is in %gs's slot on the stack */ 1196ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo pushl %fs 1197ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo CFI_ADJUST_CFA_OFFSET 4 1198ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo /*CFI_REL_OFFSET fs, 0*/ 1199d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %es 1200d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1201d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /*CFI_REL_OFFSET es, 0*/ 1202d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %ds 1203d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1204d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /*CFI_REL_OFFSET ds, 0*/ 1205d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %eax 1206d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1207d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET eax, 0 1208d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %ebp 1209d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1210d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET ebp, 0 1211d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %edi 1212d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1213d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET edi, 0 1214d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %esi 1215d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1216d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET esi, 0 1217d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %edx 1218d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1219d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET edx, 0 1220d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %ecx 1221d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1222d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET ecx, 0 1223d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %ebx 1224d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1225d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET ebx, 0 1226d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cld 1227d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl $(__KERNEL_PERCPU), %ecx 1228d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %ecx, %fs 1229d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum UNWIND_ESPFIX_STACK 1230ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo GS_TO_REG %ecx 1231ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo movl PT_GS(%esp), %edi # get the function address 1232d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl PT_ORIG_EAX(%esp), %edx # get the error code 1233d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart 1234ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo REG_TO_PTGS %ecx 1235ccbeed3a05908d201b47b6c3dd1a373138bba566Tejun Heo SET_KERNEL_GS %ecx 1236d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl $(__USER_DS), %ecx 1237d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %ecx, %ds 1238d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %ecx, %es 1239d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum TRACE_IRQS_OFF 1240d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %esp,%eax # pt_regs pointer 1241d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum call *%edi 1242d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp ret_from_exception 1243d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ENDPROC 1244d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumEND(page_fault) 1245d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1246d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum/* 1247d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * Debug traps and NMI can happen at the one SYSENTER instruction 1248d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * that sets up the real kernel stack. Check here, since we can't 1249d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * allow the wrong stack to be used. 1250d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * 1251d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have 1252d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * already pushed 3 words if it hits on the sysenter instruction: 1253d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * eflags, cs and eip. 1254d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * 1255d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * We just load the right stack, and push the three (known) values 1256d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * by hand onto the new stack - while updating the return eip past 1257d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * the instruction that would have done it for sysenter. 1258d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum */ 1259f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro FIX_STACK offset ok label 1260f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo cmpw $__KERNEL_CS, 4(%esp) 1261f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo jne \ok 1262f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo\label: 1263f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl TSS_sysenter_sp0 + \offset(%esp), %esp 1264f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_DEF_CFA esp, 0 1265f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_UNDEFINED eip 1266f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushfl 1267f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 1268f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl $__KERNEL_CS 1269f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 1270f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl $sysenter_past_esp 1271f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 1272d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET eip, 0 1273f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 1274d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1275d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumENTRY(debug) 1276d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum RING0_INT_FRAME 1277d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpl $ia32_sysenter_target,(%esp) 1278d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jne debug_stack_correct 1279f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn 1280d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelumdebug_stack_correct: 1281d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl $-1 # mark this as an int 1282d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1283d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum SAVE_ALL 1284d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum TRACE_IRQS_OFF 1285d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum xorl %edx,%edx # error code 0 1286d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %esp,%eax # pt_regs pointer 1287d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum call do_debug 1288d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp ret_from_exception 1289d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ENDPROC 1290d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumEND(debug) 1291d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1292d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum/* 1293d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * NMI is doubly nasty. It can happen _while_ we're handling 1294d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * a debug fault, and the debug fault hasn't yet been able to 1295d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * clear up the stack. So we first check whether we got an 1296d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * NMI on the sysenter entry path, but after that we need to 1297d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * check whether we got an NMI on the debug path where the debug 1298d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * fault happened on the sysenter path. 1299d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum */ 1300d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumENTRY(nmi) 1301d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum RING0_INT_FRAME 1302d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %eax 1303d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1304d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %ss, %eax 1305d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpw $__ESPFIX_SS, %ax 1306d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum popl %eax 1307d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET -4 1308d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum je nmi_espfix_stack 1309d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpl $ia32_sysenter_target,(%esp) 1310d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum je nmi_stack_fixup 1311d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %eax 1312d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1313d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %esp,%eax 1314d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /* Do not access memory above the end of our stack page, 1315d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * it might not exist. 1316d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum */ 1317d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum andl $(THREAD_SIZE-1),%eax 1318d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpl $(THREAD_SIZE-20),%eax 1319d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum popl %eax 1320d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET -4 1321d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jae nmi_stack_correct 1322d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpl $ia32_sysenter_target,12(%esp) 1323d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum je nmi_debug_stack_check 1324d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelumnmi_stack_correct: 1325d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /* We have a RING0_INT_FRAME here */ 1326d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %eax 1327d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1328d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum SAVE_ALL 1329d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum xorl %edx,%edx # zero error code 1330d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %esp,%eax # pt_regs pointer 1331d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum call do_nmi 1332d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp restore_nocheck_notrace 1333d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ENDPROC 1334d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1335d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelumnmi_stack_fixup: 1336d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum RING0_INT_FRAME 1337f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo FIX_STACK 12, nmi_stack_correct, 1 1338d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp nmi_stack_correct 1339d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1340d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelumnmi_debug_stack_check: 1341d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /* We have a RING0_INT_FRAME here */ 1342d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpw $__KERNEL_CS,16(%esp) 1343d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jne nmi_stack_correct 1344d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpl $debug,(%esp) 1345d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jb nmi_stack_correct 1346d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpl $debug_esp_fix_insn,(%esp) 1347d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum ja nmi_stack_correct 1348f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo FIX_STACK 24, nmi_stack_correct, 1 1349d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp nmi_stack_correct 1350d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1351d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelumnmi_espfix_stack: 1352d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /* We have a RING0_INT_FRAME here. 1353d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * 1354d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * create the pointer to lss back 1355d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum */ 1356d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %ss 1357d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1358d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %esp 1359d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1360bda3a89745d7bb88d3e735046c0cdc3d0eb2ac24Stas Sergeev addl $4, (%esp) 1361d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /* copy the iret frame of 12 bytes */ 1362d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum .rept 3 1363d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl 16(%esp) 1364d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1365d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum .endr 1366d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %eax 1367d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1368d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum SAVE_ALL 1369d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum FIXUP_ESPFIX_STACK # %eax == %esp 1370d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum xorl %edx,%edx # zero error code 1371d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum call do_nmi 1372d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum RESTORE_REGS 1373d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum lss 12+4(%esp), %esp # back to espfix stack 1374d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET -24 1375d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp irq_return 1376d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ENDPROC 1377d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumEND(nmi) 1378d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1379d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumENTRY(int3) 1380d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum RING0_INT_FRAME 1381d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl $-1 # mark this as an int 1382d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1383d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum SAVE_ALL 1384d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum TRACE_IRQS_OFF 1385d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum xorl %edx,%edx # zero error code 1386d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %esp,%eax # pt_regs pointer 1387d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum call do_int3 1388d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp ret_from_exception 1389d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ENDPROC 1390d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumEND(int3) 1391d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1392d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumENTRY(general_protection) 1393d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum RING0_EC_FRAME 1394d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl $do_general_protection 1395d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1396d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp error_code 1397d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ENDPROC 1398d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumEND(general_protection) 1399d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1400d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum/* 1401d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * End of kprobes section 1402d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum */ 1403d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum .popsection 1404