entry_32.S revision e7d3737ea1b102030f44e96c97754101e41515f0
11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (C) 1991, 1992 Linus Torvalds 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * entry.S contains the system-call and fault low-level handling routines. 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This also contains the timer-interrupt handler, as well as all interrupts 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * and faults that can result in a task-switch. 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * NOTE: This code handles signal-recognition, which happens every time 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * after a timer-interrupt and after each system call. 131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * I changed all the .align's to 4 (16 byte alignment), as that's faster 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * on a 486. 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 17889f21ce272e38db19c8114a7e0a5793d4590077Andi Kleen * Stack layout in 'syscall_exit': 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * ptrace needs to have all regs on the stack. 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * if the order here is changed, it needs to be 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * updated in fork.c:copy_process, signal.c:do_signal, 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * ptrace.c and ptrace.h 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 0(%esp) - %ebx 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 4(%esp) - %ecx 251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 8(%esp) - %edx 261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * C(%esp) - %esi 271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 10(%esp) - %edi 281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 14(%esp) - %ebp 291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 18(%esp) - %eax 301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1C(%esp) - %ds 311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 20(%esp) - %es 32464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge * 24(%esp) - %fs 33f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge * 28(%esp) - orig_eax 34f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge * 2C(%esp) - %eip 35f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge * 30(%esp) - %cs 36f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge * 34(%esp) - %eflags 37f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge * 38(%esp) - %oldesp 38f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge * 3C(%esp) - %oldss 391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * "current" is in register %ebx during any slow entries. 411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/linkage.h> 441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/thread_info.h> 4555f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar#include <asm/irqflags.h> 461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/errno.h> 471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/segment.h> 481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/smp.h> 491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/page.h> 501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/desc.h> 51be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev#include <asm/percpu.h> 52fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich#include <asm/dwarf2.h> 53ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov#include <asm/processor-flags.h> 54395a59d0f8e86bb39cd700c3d185d30c670bb958Abhishek Sagar#include <asm/ftrace.h> 559b7dc567d03d74a1fbae84e88949b6a60d922d82Thomas Gleixner#include <asm/irq_vectors.h> 561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 57af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */ 58af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#include <linux/elf-em.h> 59af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#define AUDIT_ARCH_I386 (EM_386|__AUDIT_ARCH_LE) 60af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#define __AUDIT_ARCH_LE 0x40000000 61af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath 62af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#ifndef CONFIG_AUDITSYSCALL 63af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#define sysenter_audit syscall_trace_entry 64af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#define sysexit_audit syscall_exit_work 65af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#endif 66af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath 67139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell/* 68139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * We use macros for low-level operations which need to be overridden 69139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * for paravirtualization. The following will never clobber any registers: 70139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * INTERRUPT_RETURN (aka. "iret") 71139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") 72d75cd22fdd5f7d203fb60014d426942df33dd9a6Jeremy Fitzhardinge * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit"). 73139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * 74139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must 75139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). 76139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * Allowing a register to be clobbered can shrink the paravirt replacement 77139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * enough to patch inline, increasing performance. 78139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell */ 79139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell 801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define nr_syscalls ((syscall_table_size)/4) 811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef CONFIG_PREEMPT 83139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell#define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF 841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#else 85139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell#define preempt_stop(clobbers) 861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define resume_kernel restore_nocheck 871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8955f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar.macro TRACE_IRQS_IRET 9055f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar#ifdef CONFIG_TRACE_IRQFLAGS 91ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off? 9255f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar jz 1f 9355f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_ON 9455f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar1: 9555f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar#endif 9655f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar.endm 9755f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar 984031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov#ifdef CONFIG_VM86 994031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov#define resume_userspace_sig check_userspace 1004031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov#else 1014031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov#define resume_userspace_sig resume_userspace 1024031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov#endif 1034031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov 1041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define SAVE_ALL \ 1051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cld; \ 106464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge pushl %fs; \ 107f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET 4;\ 108464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge /*CFI_REL_OFFSET fs, 0;*/\ 1091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %es; \ 110fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4;\ 111fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_REL_OFFSET es, 0;*/\ 1121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ds; \ 113fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4;\ 114fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_REL_OFFSET ds, 0;*/\ 1151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax; \ 116fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4;\ 117fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET eax, 0;\ 1181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ebp; \ 119fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4;\ 120fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET ebp, 0;\ 1211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %edi; \ 122fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4;\ 123fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET edi, 0;\ 1241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %esi; \ 125fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4;\ 126fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET esi, 0;\ 1271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %edx; \ 128fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4;\ 129fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET edx, 0;\ 1301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ecx; \ 131fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4;\ 132fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET ecx, 0;\ 1331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ebx; \ 134fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4;\ 135fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET ebx, 0;\ 1361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl $(__USER_DS), %edx; \ 1371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %edx, %ds; \ 138f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge movl %edx, %es; \ 1397c3576d261ce046789a7db14f43303f8120910c7Jeremy Fitzhardinge movl $(__KERNEL_PERCPU), %edx; \ 140464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge movl %edx, %fs 1411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define RESTORE_INT_REGS \ 1431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %ebx; \ 144fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4;\ 145fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_RESTORE ebx;\ 1461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %ecx; \ 147fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4;\ 148fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_RESTORE ecx;\ 1491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %edx; \ 150fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4;\ 151fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_RESTORE edx;\ 1521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %esi; \ 153fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4;\ 154fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_RESTORE esi;\ 1551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %edi; \ 156fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4;\ 157fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_RESTORE edi;\ 1581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %ebp; \ 159fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4;\ 160fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_RESTORE ebp;\ 161fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich popl %eax; \ 162fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4;\ 163fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_RESTORE eax 1641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define RESTORE_REGS \ 1661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds RESTORE_INT_REGS; \ 1671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds1: popl %ds; \ 168fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4;\ 169fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_RESTORE ds;*/\ 1701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds2: popl %es; \ 171fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4;\ 172fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_RESTORE es;*/\ 173464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge3: popl %fs; \ 174f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET -4;\ 175464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge /*CFI_RESTORE fs;*/\ 176f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge.pushsection .fixup,"ax"; \ 1771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds4: movl $0,(%esp); \ 178f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge jmp 1b; \ 179f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge5: movl $0,(%esp); \ 1801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp 2b; \ 181f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge6: movl $0,(%esp); \ 182f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge jmp 3b; \ 1831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.section __ex_table,"a";\ 1841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .align 4; \ 185f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge .long 1b,4b; \ 186f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge .long 2b,5b; \ 187f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge .long 3b,6b; \ 188f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge.popsection 1891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 190fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich#define RING0_INT_FRAME \ 191fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_STARTPROC simple;\ 192adf1423698f00d00b267f7dca8231340ce7d65efJan Beulich CFI_SIGNAL_FRAME;\ 193fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_DEF_CFA esp, 3*4;\ 194fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_OFFSET cs, -2*4;*/\ 195fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_OFFSET eip, -3*4 196fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich 197fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich#define RING0_EC_FRAME \ 198fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_STARTPROC simple;\ 199adf1423698f00d00b267f7dca8231340ce7d65efJan Beulich CFI_SIGNAL_FRAME;\ 200fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_DEF_CFA esp, 4*4;\ 201fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_OFFSET cs, -2*4;*/\ 202fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_OFFSET eip, -3*4 203fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich 204fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich#define RING0_PTREGS_FRAME \ 205fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_STARTPROC simple;\ 206adf1423698f00d00b267f7dca8231340ce7d65efJan Beulich CFI_SIGNAL_FRAME;\ 207eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge CFI_DEF_CFA esp, PT_OLDESP-PT_EBX;\ 208eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/\ 209eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge CFI_OFFSET eip, PT_EIP-PT_OLDESP;\ 210eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/\ 211eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/\ 212eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge CFI_OFFSET eax, PT_EAX-PT_OLDESP;\ 213eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge CFI_OFFSET ebp, PT_EBP-PT_OLDESP;\ 214eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge CFI_OFFSET edi, PT_EDI-PT_OLDESP;\ 215eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge CFI_OFFSET esi, PT_ESI-PT_OLDESP;\ 216eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge CFI_OFFSET edx, PT_EDX-PT_OLDESP;\ 217eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge CFI_OFFSET ecx, PT_ECX-PT_OLDESP;\ 218eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge CFI_OFFSET ebx, PT_EBX-PT_OLDESP 2191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(ret_from_fork) 221fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_STARTPROC 2221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax 22325d7dfdaf3404bb31b8f55283fd2c456cb7b4001Markus Armbruster CFI_ADJUST_CFA_OFFSET 4 2241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call schedule_tail 2251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 2261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %eax 227fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 22847a5c6fa0e204a2b63309c648bb2fde36836c826Linus Torvalds pushl $0x0202 # Reset kernel eflags 22947a5c6fa0e204a2b63309c648bb2fde36836c826Linus Torvalds CFI_ADJUST_CFA_OFFSET 4 23047a5c6fa0e204a2b63309c648bb2fde36836c826Linus Torvalds popfl 23147a5c6fa0e204a2b63309c648bb2fde36836c826Linus Torvalds CFI_ADJUST_CFA_OFFSET -4 2321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp syscall_exit 233fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 23447a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(ret_from_fork) 2351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 2371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Return to user mode is not as complex as all this looks, 2381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * but we want the default path for a system call return to 2391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * go as quickly as possible which is why some of this is 2401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * less clear than it otherwise should be. 2411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 2421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # userspace resumption stub bypassing syscall exit tracing 2441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 245fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_PTREGS_FRAME 2461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsret_from_exception: 247139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell preempt_stop(CLBR_ANY) 2481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsret_from_intr: 2491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 2504031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelovcheck_userspace: 251eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS 252eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movb PT_CS(%esp), %al 253ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax 25478be3706b21a232310590fe00258b224177ac05fRusty Russell cmpl $USER_RPL, %eax 25578be3706b21a232310590fe00258b224177ac05fRusty Russell jb resume_kernel # not returning to v8086 or userspace 256f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge 2571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(resume_userspace) 258c7e872e7da5514d014707a407ea562d197cc0136Peter Zijlstra LOCKDEP_SYS_EXIT 259139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 2601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # setting need_resched or sigpending 2611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # between sampling and the iret 262e32e58a96de4ac35a03349db2ab69f263ded958fPeter Zijlstra TRACE_IRQS_OFF 2631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx 2641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds andl $_TIF_WORK_MASK, %ecx # is there any work to be done on 2651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # int/exception return? 2661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne work_pending 2671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp restore_all 26847a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(ret_from_exception) 2691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef CONFIG_PREEMPT 2711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(resume_kernel) 272139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_ANY) 2731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ? 2741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz restore_nocheck 2751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsneed_resched: 2761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx # need_resched set ? 2771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testb $_TIF_NEED_RESCHED, %cl 2781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz restore_all 279ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off (exception path) ? 2801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz restore_all 2811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call preempt_schedule_irq 2821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp need_resched 28347a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(resume_kernel) 2841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 285fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 2861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* SYSENTER_RETURN points to after the "sysenter" instruction in 2881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */ 2891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # sysenter call handler stub 2910aa97fb22624f18e5925d702ab0364d3838cfd91Roland McGrathENTRY(ia32_sysenter_target) 292fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_STARTPROC simple 293adf1423698f00d00b267f7dca8231340ce7d65efJan Beulich CFI_SIGNAL_FRAME 294fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_DEF_CFA esp, 0 295fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REGISTER esp, ebp 296faca62273b602ab482fb7d3d940dbf41ef08b00eH. Peter Anvin movl TSS_sysenter_sp0(%esp),%esp 2971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssysenter_past_esp: 29855f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar /* 299d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge * Interrupts are disabled here, but we can't trace it until 300d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge * enough kernel state to call TRACE_IRQS_OFF can be called - but 301d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge * we immediately enable interrupts at that point anyway. 30255f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar */ 3031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $(__USER_DS) 304fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 305fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_REL_OFFSET ss, 0*/ 3061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ebp 307fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 308fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET esp, 0 3091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushfl 310d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge orl $X86_EFLAGS_IF, (%esp) 311fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 3121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $(__USER_CS) 313fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 314fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_REL_OFFSET cs, 0*/ 315e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar /* 316e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar * Push current_thread_info()->sysenter_return to the stack. 317e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar * A tiny bit of offset fixup is necessary - 4*4 means the 4 words 318e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar * pushed above; +8 corresponds to copy_thread's esp0 setting. 319e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar */ 320e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp) 321fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 322fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET eip, 0 3231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 324d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge pushl %eax 325d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET 4 326d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge SAVE_ALL 327d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge ENABLE_INTERRUPTS(CLBR_NONE) 328d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge 3291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 3301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Load the potential sixth argument from user stack. 3311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Careful about security. 3321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 3331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $__PAGE_OFFSET-3,%ebp 3341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jae syscall_fault 3351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds1: movl (%ebp),%ebp 336d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge movl %ebp,PT_EBP(%esp) 3371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.section __ex_table,"a" 3381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .align 4 3391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .long 1b,syscall_fault 3401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.previous 3411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 3431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */ 345d4d67150165df8bf1cc05e532f6efca96f907cabRoland McGrath testw $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp) 346af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jnz sysenter_audit 347af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrathsysenter_do_call: 3481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $(nr_syscalls), %eax 3491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jae syscall_badsys 3501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call *sys_call_table(,%eax,4) 351eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl %eax,PT_EAX(%esp) 352c7e872e7da5514d014707a407ea562d197cc0136Peter Zijlstra LOCKDEP_SYS_EXIT 35342c24fa22e86365055fc931d833f26165e687c19Jeremy Fitzhardinge DISABLE_INTERRUPTS(CLBR_ANY) 35455f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 3551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx 3561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testw $_TIF_ALLWORK_MASK, %cx 357af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jne sysexit_audit 358af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrathsysenter_exit: 3591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* if something modifies registers it must also disable sysexit */ 360eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_EIP(%esp), %edx 361eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_OLDESP(%esp), %ecx 3621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %ebp,%ebp 36355f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_ON 364464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge1: mov PT_FS(%esp), %fs 365d75cd22fdd5f7d203fb60014d426942df33dd9a6Jeremy Fitzhardinge ENABLE_INTERRUPTS_SYSEXIT 366af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath 367af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#ifdef CONFIG_AUDITSYSCALL 368af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrathsysenter_audit: 369af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath testw $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%ebp) 370af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jnz syscall_trace_entry 371af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath addl $4,%esp 372af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath CFI_ADJUST_CFA_OFFSET -4 373af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath /* %esi already in 8(%esp) 6th arg: 4th syscall arg */ 374af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath /* %edx already in 4(%esp) 5th arg: 3rd syscall arg */ 375af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath /* %ecx already in 0(%esp) 4th arg: 2nd syscall arg */ 376af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl %ebx,%ecx /* 3rd arg: 1st syscall arg */ 377af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl %eax,%edx /* 2nd arg: syscall number */ 378af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl $AUDIT_ARCH_I386,%eax /* 1st arg: audit arch */ 379af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath call audit_syscall_entry 380af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath pushl %ebx 381af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath CFI_ADJUST_CFA_OFFSET 4 382af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl PT_EAX(%esp),%eax /* reload syscall number */ 383af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jmp sysenter_do_call 384af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath 385af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrathsysexit_audit: 386af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath testw $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %cx 387af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jne syscall_exit_work 388af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath TRACE_IRQS_ON 389af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath ENABLE_INTERRUPTS(CLBR_ANY) 390af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl %eax,%edx /* second arg, syscall return value */ 391af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath cmpl $0,%eax /* is it < 0? */ 392af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath setl %al /* 1 if so, 0 if not */ 393af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movzbl %al,%eax /* zero-extend that */ 394af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath inc %eax /* first arg, 0->1(AUDITSC_SUCCESS), 1->2(AUDITSC_FAILURE) */ 395af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath call audit_syscall_exit 396af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath DISABLE_INTERRUPTS(CLBR_ANY) 397af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath TRACE_IRQS_OFF 398af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl TI_flags(%ebp), %ecx 399af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath testw $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %cx 400af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jne syscall_exit_work 401af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl PT_EAX(%esp),%eax /* reload syscall return value */ 402af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jmp sysenter_exit 403af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#endif 404af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath 405fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 406f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge.pushsection .fixup,"ax" 407464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge2: movl $0,PT_FS(%esp) 408f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge jmp 1b 409f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge.section __ex_table,"a" 410f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge .align 4 411f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge .long 1b,2b 412f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge.popsection 4130aa97fb22624f18e5925d702ab0364d3838cfd91Roland McGrathENDPROC(ia32_sysenter_target) 4141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # system call handler stub 4161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(system_call) 417fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME # can't unwind into user space anyway 4181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax # save orig_eax 419fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 4201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds SAVE_ALL 4211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 422ed75e8d58010fdc06e2c3a81bfbebae92314c7e3Laurent Vivier # system call tracing in operation / emulation 4231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */ 424d4d67150165df8bf1cc05e532f6efca96f907cabRoland McGrath testw $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp) 4251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz syscall_trace_entry 4261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $(nr_syscalls), %eax 4271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jae syscall_badsys 4281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_call: 4291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call *sys_call_table(,%eax,4) 430eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl %eax,PT_EAX(%esp) # store the return value 4311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_exit: 432c7e872e7da5514d014707a407ea562d197cc0136Peter Zijlstra LOCKDEP_SYS_EXIT 433139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 4341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # setting need_resched or sigpending 4351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # between sampling and the iret 43655f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 4371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx 4381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testw $_TIF_ALLWORK_MASK, %cx # current->work 4391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne syscall_exit_work 4401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsrestore_all: 442eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS 443eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge # Warning: PT_OLDSS(%esp) contains the wrong/random values if we 4445df240826c90afdc7956f55a004ea6b702df9203Stas Sergeev # are returning to the kernel. 4455df240826c90afdc7956f55a004ea6b702df9203Stas Sergeev # See comments in process.c:copy_thread() for details. 446eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movb PT_OLDSS(%esp), %ah 447eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movb PT_CS(%esp), %al 448ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax 44978be3706b21a232310590fe00258b224177ac05fRusty Russell cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax 450fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REMEMBER_STATE 4511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds je ldt_ss # returning to user-space with LDT SS 4521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsrestore_nocheck: 45355f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_IRET 45455f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnarrestore_nocheck_notrace: 4551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds RESTORE_REGS 456f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge addl $4, %esp # skip orig_eax/error_code 457fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 458f7f3d791e61d7baf8b0aee0384fdd469c0d2ac9bAdrian Bunkirq_return: 4593701d863b43d05ffeb223d269583398f914fb5d3Ingo Molnar INTERRUPT_RETURN 4601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.section .fixup,"ax" 46190e9f53662826db3cdd6d99bd394d727b05160c1Jeremy FitzhardingeENTRY(iret_exc) 462a879cbbb34cbecfa9707fbb6e5a00c503ac1ecb9Linus Torvalds pushl $0 # no error code 463a879cbbb34cbecfa9707fbb6e5a00c503ac1ecb9Linus Torvalds pushl $do_iret_error 464a879cbbb34cbecfa9707fbb6e5a00c503ac1ecb9Linus Torvalds jmp error_code 4651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.previous 4661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.section __ex_table,"a" 4671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .align 4 4683701d863b43d05ffeb223d269583398f914fb5d3Ingo Molnar .long irq_return,iret_exc 4691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.previous 4701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 471fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_RESTORE_STATE 4721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsldt_ss: 473eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge larl PT_OLDSS(%esp), %eax 4741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz restore_nocheck 4751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testl $0x00400000, %eax # returning to 32bit stack? 4761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz restore_nocheck # allright, normal return 477d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell 478d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell#ifdef CONFIG_PARAVIRT 479d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell /* 480d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * The kernel can't run on a non-flat stack if paravirt mode 481d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * is active. Rather than try to fixup the high bits of 482d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * ESP, bypass this code entirely. This may break DOSemu 483d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * and/or Wine support in a paravirt VM, although the option 484d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * is still available to implement the setting of the high 485d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * 16-bits in the INTERRUPT_RETURN paravirt-op. 486d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell */ 48793b1eab3d29e7ea32ee583de3362da84db06ded8Jeremy Fitzhardinge cmpl $0, pv_info+PARAVIRT_enabled 488d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell jne restore_nocheck 489d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell#endif 490d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell 4911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* If returning to userspace with 16bit stack, 4921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * try to fix the higher word of ESP, as the CPU 4931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * won't restore it. 4941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This is an "official" bug of all the x86-compatible 4951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * CPUs, which we can try to work around to make 4961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * dosemu and wine happy. */ 497eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_OLDESP(%esp), %eax 498be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev movl %esp, %edx 499be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev call patch_espfix_desc 500be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev pushl $__ESPFIX_SS 501be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev CFI_ADJUST_CFA_OFFSET 4 502be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev pushl %eax 503be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev CFI_ADJUST_CFA_OFFSET 4 504139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_EAX) 50555f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 506be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev lss (%esp), %esp 507be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev CFI_ADJUST_CFA_OFFSET -8 508be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev jmp restore_nocheck 509fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 51047a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichENDPROC(system_call) 5111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # perform work that needs to be done immediately before resumption 5131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 514fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_PTREGS_FRAME # can't unwind into user space anyway 5151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldswork_pending: 5161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testb $_TIF_NEED_RESCHED, %cl 5171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz work_notifysig 5181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldswork_resched: 5191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call schedule 520c7e872e7da5514d014707a407ea562d197cc0136Peter Zijlstra LOCKDEP_SYS_EXIT 521139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 5221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # setting need_resched or sigpending 5231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # between sampling and the iret 52455f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 5251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx 5261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds andl $_TIF_WORK_MASK, %ecx # is there any work to be done other 5271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # than syscall tracing? 5281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz restore_all 5291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testb $_TIF_NEED_RESCHED, %cl 5301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz work_resched 5311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldswork_notifysig: # deal with pending signals and 5331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # notify-resume requests 53474b47a7844501445d41d704fe7c626f4b1819508Joe Korty#ifdef CONFIG_VM86 535ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov testl $X86_EFLAGS_VM, PT_EFLAGS(%esp) 5361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp, %eax 5371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne work_notifysig_v86 # returning to kernel-space or 5381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # vm86-space 5391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %edx, %edx 5401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_notify_resume 5414031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov jmp resume_userspace_sig 5421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 5441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldswork_notifysig_v86: 5451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ecx # save ti_flags for do_notify_resume 546fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 5471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call save_v86_state # %eax contains pt_regs pointer 5481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %ecx 549fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 5501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %eax, %esp 55174b47a7844501445d41d704fe7c626f4b1819508Joe Korty#else 55274b47a7844501445d41d704fe7c626f4b1819508Joe Korty movl %esp, %eax 55374b47a7844501445d41d704fe7c626f4b1819508Joe Korty#endif 5541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %edx, %edx 5551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_notify_resume 5564031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov jmp resume_userspace_sig 55747a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(work_pending) 5581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # perform syscall exit tracing 5601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 5611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_trace_entry: 562eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl $-ENOSYS,PT_EAX(%esp) 5631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp, %eax 564d4d67150165df8bf1cc05e532f6efca96f907cabRoland McGrath call syscall_trace_enter 565d4d67150165df8bf1cc05e532f6efca96f907cabRoland McGrath /* What it returned is what we'll actually use. */ 5661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $(nr_syscalls), %eax 5671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnae syscall_call 5681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp syscall_exit 56947a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(syscall_trace_entry) 5701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # perform syscall exit tracing 5721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 5731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_exit_work: 574d4d67150165df8bf1cc05e532f6efca96f907cabRoland McGrath testb $_TIF_WORK_SYSCALL_EXIT, %cl 5751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz work_pending 57655f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_ON 577d4d67150165df8bf1cc05e532f6efca96f907cabRoland McGrath ENABLE_INTERRUPTS(CLBR_ANY) # could let syscall_trace_leave() call 5781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # schedule() instead 5791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp, %eax 580d4d67150165df8bf1cc05e532f6efca96f907cabRoland McGrath call syscall_trace_leave 5811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp resume_userspace 58247a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(syscall_exit_work) 583fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 5841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 585fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME # can't unwind into user space anyway 5861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_fault: 5871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 588eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl $-EFAULT,PT_EAX(%esp) 5891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp resume_userspace 59047a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(syscall_fault) 5911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_badsys: 593eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl $-ENOSYS,PT_EAX(%esp) 5941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp resume_userspace 59547a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(syscall_badsys) 596fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 5971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define FIXUP_ESPFIX_STACK \ 599be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev /* since we are on a wrong stack, we cant make it a C code :( */ \ 6007a61d35d4b4056e7711031202da7605e052f4137Jeremy Fitzhardinge PER_CPU(gdt_page, %ebx); \ 601be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah); \ 602be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev addl %esp, %eax; \ 603be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev pushl $__KERNEL_DS; \ 604be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev CFI_ADJUST_CFA_OFFSET 4; \ 6051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax; \ 606fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4; \ 607be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev lss (%esp), %esp; \ 608be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev CFI_ADJUST_CFA_OFFSET -8; 609be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev#define UNWIND_ESPFIX_STACK \ 6101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %ss, %eax; \ 611be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev /* see if on espfix stack */ \ 6121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpw $__ESPFIX_SS, %ax; \ 613be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev jne 27f; \ 614be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev movl $__KERNEL_DS, %eax; \ 615fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich movl %eax, %ds; \ 616fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich movl %eax, %es; \ 617be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev /* switch to normal stack */ \ 618fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich FIXUP_ESPFIX_STACK; \ 619be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev27:; 6201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 6221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Build the entry stubs and pointer table with 6231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * some assembler magic. 6241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 6253e7622f9d7807a0a826d042cafc211cd1a29448cJan Beulich.section .rodata,"a" 6261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(interrupt) 6271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.text 6281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(irq_entries_start) 630fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 63147a55cd795656d11bb18a7885583361f02a6baa8Jan Beulichvector=0 632497c9a195db918d3f035e8cb3021e5d4d035516eYinghai Lu.rept NR_VECTORS 6331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 634fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich .if vector 635fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 636fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich .endif 63719eadf98c8167eac843580683317b99572e2abf0Rusty Russell1: pushl $~(vector) 638fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 6391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp common_interrupt 64047a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich .previous 6411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .long 1b 64247a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich .text 6431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvector=vector+1 6441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.endr 64547a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(irq_entries_start) 64647a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich 64747a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich.previous 64847a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(interrupt) 64947a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich.previous 6501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 65155f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar/* 65255f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar * the CPU automatically disables interrupts when executing an IRQ vector, 65355f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar * so IRQ-flags tracing has to follow that: 65455f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar */ 6551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 6561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldscommon_interrupt: 6571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds SAVE_ALL 65855f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 6591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp,%eax 6601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_IRQ 6611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp ret_from_intr 66247a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichENDPROC(common_interrupt) 663fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 6641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define BUILD_INTERRUPT(name, nr) \ 6661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(name) \ 667fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME; \ 66819eadf98c8167eac843580683317b99572e2abf0Rusty Russell pushl $~(nr); \ 669fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4; \ 670fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich SAVE_ALL; \ 67155f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF \ 6721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp,%eax; \ 673f76c392380a40008ee6ecaea4e5a51a3a10282c4Jeremy Fitzhardinge call smp_##name; \ 67455f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar jmp ret_from_intr; \ 67547a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich CFI_ENDPROC; \ 67647a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichENDPROC(name) 6771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* The include is where all of the SMP etc. interrupts come from */ 6791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include "entry_arch.h" 6801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 681d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.PKPROBE_ENTRY(page_fault) 682d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P RING0_EC_FRAME 683d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P pushl $do_page_fault 684fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 6851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 6861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldserror_code: 687464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge /* the function address is in %fs's slot on the stack */ 688f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge pushl %es 689f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET 4 690f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge /*CFI_REL_OFFSET es, 0*/ 6911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ds 692fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 693fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_REL_OFFSET ds, 0*/ 6941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax 695fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 696fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET eax, 0 6971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ebp 698fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 699fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET ebp, 0 7001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %edi 701fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 702fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET edi, 0 7031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %esi 704fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 705fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET esi, 0 7061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %edx 707fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 708fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET edx, 0 7091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ecx 710fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 711fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET ecx, 0 7121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ebx 713fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 714fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET ebx, 0 7151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cld 716464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge pushl %fs 717fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 718464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge /*CFI_REL_OFFSET fs, 0*/ 7197c3576d261ce046789a7db14f43303f8120910c7Jeremy Fitzhardinge movl $(__KERNEL_PERCPU), %ecx 720464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge movl %ecx, %fs 7211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds UNWIND_ESPFIX_STACK 7221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %ecx 723fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 724fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_REGISTER es, ecx*/ 725464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge movl PT_FS(%esp), %edi # get the function address 726eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_ORIG_EAX(%esp), %edx # get the error code 727f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart 728464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge mov %ecx, PT_FS(%esp) 729464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge /*CFI_REL_OFFSET fs, ES*/ 7301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl $(__USER_DS), %ecx 7311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %ecx, %ds 7321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %ecx, %es 73385cea51d7e7b8d3408c8e933d88fa067309395faAlexander van Heukelum TRACE_IRQS_OFF 7341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp,%eax # pt_regs pointer 7351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call *%edi 7361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp ret_from_exception 737fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 738d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.PKPROBE_END(page_fault) 7391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(coprocessor_error) 741fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 7421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 743fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_coprocessor_error 745fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 747fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 74847a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(coprocessor_error) 7491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(simd_coprocessor_error) 751fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 7521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 753fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_simd_coprocessor_error 755fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 757fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 75847a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(simd_coprocessor_error) 7591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(device_not_available) 761fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 7621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $-1 # mark this as an int 763fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7647643e9b936b4af31ba4851eb7d5b3a3bfad52502Alexander van Heukelum pushl $do_device_not_available 765fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7667643e9b936b4af31ba4851eb7d5b3a3bfad52502Alexander van Heukelum jmp error_code 767fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 76847a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(device_not_available) 7691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 7711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Debug traps and NMI can happen at the one SYSENTER instruction 7721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * that sets up the real kernel stack. Check here, since we can't 7731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * allow the wrong stack to be used. 7741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 775faca62273b602ab482fb7d3d940dbf41ef08b00eH. Peter Anvin * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have 7761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * already pushed 3 words if it hits on the sysenter instruction: 7771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * eflags, cs and eip. 7781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 7791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * We just load the right stack, and push the three (known) values 7801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * by hand onto the new stack - while updating the return eip past 7811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * the instruction that would have done it for sysenter. 7821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 7831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define FIX_STACK(offset, ok, label) \ 7841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpw $__KERNEL_CS,4(%esp); \ 7851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne ok; \ 7861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldslabel: \ 787faca62273b602ab482fb7d3d940dbf41ef08b00eH. Peter Anvin movl TSS_sysenter_sp0+offset(%esp),%esp; \ 788a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert CFI_DEF_CFA esp, 0; \ 789a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert CFI_UNDEFINED eip; \ 7901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushfl; \ 791a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert CFI_ADJUST_CFA_OFFSET 4; \ 7921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $__KERNEL_CS; \ 793a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert CFI_ADJUST_CFA_OFFSET 4; \ 794a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert pushl $sysenter_past_esp; \ 795a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert CFI_ADJUST_CFA_OFFSET 4; \ 796a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert CFI_REL_OFFSET eip, 0 7971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7983d97ae5b958855ac007b6f56a0f94ab8ade09e9ePrasanna S PanchamukhiKPROBE_ENTRY(debug) 799fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 8000aa97fb22624f18e5925d702ab0364d3838cfd91Roland McGrath cmpl $ia32_sysenter_target,(%esp) 8011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne debug_stack_correct 8021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn) 8031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsdebug_stack_correct: 8041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $-1 # mark this as an int 805fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds SAVE_ALL 80743024a8a5d4c63952687286f3083f7f34d4da2ccAlexander van Heukelum TRACE_IRQS_OFF 8081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %edx,%edx # error code 0 8091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp,%eax # pt_regs pointer 8101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_debug 8111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp ret_from_exception 812fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 813d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.PKPROBE_END(debug) 814d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P 8151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 8161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * NMI is doubly nasty. It can happen _while_ we're handling 8171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * a debug fault, and the debug fault hasn't yet been able to 8181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * clear up the stack. So we first check whether we got an 8191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * NMI on the sysenter entry path, but after that we need to 8201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * check whether we got an NMI on the debug path where the debug 8211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * fault happened on the sysenter path. 8221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 82306039754d775d3e48e4a292e4f353321205eff53Fernando Luis V�zquez CaoKPROBE_ENTRY(nmi) 824fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 8251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax 826fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %ss, %eax 8281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpw $__ESPFIX_SS, %ax 8291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %eax 830fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 831be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev je nmi_espfix_stack 8320aa97fb22624f18e5925d702ab0364d3838cfd91Roland McGrath cmpl $ia32_sysenter_target,(%esp) 8331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds je nmi_stack_fixup 8341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax 835fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp,%eax 8371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Do not access memory above the end of our stack page, 8381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * it might not exist. 8391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 8401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds andl $(THREAD_SIZE-1),%eax 8411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $(THREAD_SIZE-20),%eax 8421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %eax 843fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 8441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jae nmi_stack_correct 8450aa97fb22624f18e5925d702ab0364d3838cfd91Roland McGrath cmpl $ia32_sysenter_target,12(%esp) 8461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds je nmi_debug_stack_check 8471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsnmi_stack_correct: 848a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert /* We have a RING0_INT_FRAME here */ 8491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax 850fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds SAVE_ALL 852e0c7317557c8fc8eacf611e30c2a80f4e24e47a3Alexander van Heukelum TRACE_IRQS_OFF 8531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %edx,%edx # zero error code 8541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp,%eax # pt_regs pointer 8551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_nmi 85655f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar jmp restore_nocheck_notrace 857fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 8581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsnmi_stack_fixup: 860a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert RING0_INT_FRAME 8611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds FIX_STACK(12,nmi_stack_correct, 1) 8621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp nmi_stack_correct 863a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert 8641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsnmi_debug_stack_check: 865a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert /* We have a RING0_INT_FRAME here */ 8661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpw $__KERNEL_CS,16(%esp) 8671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne nmi_stack_correct 868e27182088e607880713d9c286a3d92d861c280e4Jan Beulich cmpl $debug,(%esp) 869e27182088e607880713d9c286a3d92d861c280e4Jan Beulich jb nmi_stack_correct 8701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $debug_esp_fix_insn,(%esp) 871e27182088e607880713d9c286a3d92d861c280e4Jan Beulich ja nmi_stack_correct 8721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds FIX_STACK(24,nmi_stack_correct, 1) 8731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp nmi_stack_correct 8741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 875be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeevnmi_espfix_stack: 876a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert /* We have a RING0_INT_FRAME here. 877a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert * 878a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert * create the pointer to lss back 879a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert */ 8801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ss 881fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %esp 883fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds addw $4, (%esp) 8851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* copy the iret frame of 12 bytes */ 8861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .rept 3 8871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl 16(%esp) 888fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .endr 8901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax 891fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds SAVE_ALL 893e0c7317557c8fc8eacf611e30c2a80f4e24e47a3Alexander van Heukelum TRACE_IRQS_OFF 8941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds FIXUP_ESPFIX_STACK # %eax == %esp 8951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %edx,%edx # zero error code 8961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_nmi 8971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds RESTORE_REGS 898be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev lss 12+4(%esp), %esp # back to espfix stack 899be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev CFI_ADJUST_CFA_OFFSET -24 9003701d863b43d05ffeb223d269583398f914fb5d3Ingo Molnar jmp irq_return 901fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 90206039754d775d3e48e4a292e4f353321205eff53Fernando Luis V�zquez CaoKPROBE_END(nmi) 9031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 904d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell#ifdef CONFIG_PARAVIRT 905d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty RussellENTRY(native_iret) 9063701d863b43d05ffeb223d269583398f914fb5d3Ingo Molnar iret 907d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell.section __ex_table,"a" 908d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell .align 4 9093701d863b43d05ffeb223d269583398f914fb5d3Ingo Molnar .long native_iret, iret_exc 910d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell.previous 91147a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(native_iret) 912d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell 913d75cd22fdd5f7d203fb60014d426942df33dd9a6Jeremy FitzhardingeENTRY(native_irq_enable_sysexit) 914d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell sti 915d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell sysexit 916d75cd22fdd5f7d203fb60014d426942df33dd9a6Jeremy FitzhardingeEND(native_irq_enable_sysexit) 917d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell#endif 918d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell 9193d97ae5b958855ac007b6f56a0f94ab8ade09e9ePrasanna S PanchamukhiKPROBE_ENTRY(int3) 920fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 9211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $-1 # mark this as an int 922fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds SAVE_ALL 924a790392faa3a6138b6e90d0fe320a2829652ce22Alexander van Heukelum TRACE_IRQS_OFF 9251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %edx,%edx # zero error code 9261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp,%eax # pt_regs pointer 9271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_int3 9281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp ret_from_exception 929fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 930d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.PKPROBE_END(int3) 9311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(overflow) 933fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 9341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 935fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_overflow 937fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 939fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 94047a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(overflow) 9411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(bounds) 943fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 9441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 945fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_bounds 947fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 949fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 95047a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(bounds) 9511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(invalid_op) 953fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 9541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 955fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_invalid_op 957fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 959fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 96047a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(invalid_op) 9611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(coprocessor_segment_overrun) 963fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 9641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 965fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_coprocessor_segment_overrun 967fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 969fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 97047a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(coprocessor_segment_overrun) 9711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(invalid_TSS) 973fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_EC_FRAME 9741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_invalid_TSS 975fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 977fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 97847a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(invalid_TSS) 9791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(segment_not_present) 981fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_EC_FRAME 9821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_segment_not_present 983fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 985fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 98647a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(segment_not_present) 9871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(stack_segment) 989fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_EC_FRAME 9901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_stack_segment 991fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 993fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 99447a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(stack_segment) 9951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9963d97ae5b958855ac007b6f56a0f94ab8ade09e9ePrasanna S PanchamukhiKPROBE_ENTRY(general_protection) 997fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_EC_FRAME 9981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_general_protection 999fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 10001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 1001fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 1002d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.PKPROBE_END(general_protection) 10031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 10041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(alignment_check) 1005fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_EC_FRAME 10061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_alignment_check 1007fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 10081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 1009fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 101047a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(alignment_check) 10111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1012d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.PENTRY(divide_error) 1013d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P RING0_INT_FRAME 1014d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P pushl $0 # no error code 1015d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P CFI_ADJUST_CFA_OFFSET 4 1016d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P pushl $do_divide_error 1017fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 10181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 1019fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 102047a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(divide_error) 10211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 10221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef CONFIG_X86_MCE 10231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(machine_check) 1024fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 10251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 1026fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 1027d2f6f7aeee890df445be29a60e34925ec15f620cAndi Kleen pushl machine_check_vector 1028fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 10291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 1030fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 103147a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(machine_check) 10321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 10331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 10341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(spurious_interrupt_bug) 1035fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 10361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 1037fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 10381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_spurious_interrupt_bug 1039fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 10401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 1041fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 104247a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(spurious_interrupt_bug) 10431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 104402ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi KleenENTRY(kernel_thread_helper) 104502ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen pushl $0 # fake return address for unwinder 104602ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen CFI_STARTPROC 104702ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen movl %edx,%eax 104802ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen push %edx 104902ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen CFI_ADJUST_CFA_OFFSET 4 105002ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen call *%ebx 105102ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen push %eax 105202ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen CFI_ADJUST_CFA_OFFSET 4 105302ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen call do_exit 105402ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen CFI_ENDPROC 105502ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi KleenENDPROC(kernel_thread_helper) 105602ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen 10575ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge#ifdef CONFIG_XEN 1058e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge/* Xen doesn't set %esp to be precisely what the normal sysenter 1059e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge entrypoint expects, so fix it up before using the normal path. */ 1060e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy FitzhardingeENTRY(xen_sysenter_target) 1061e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge RING0_INT_FRAME 1062e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge addl $5*4, %esp /* remove xen-provided frame */ 10632ddf9b7b3e6660199269e34cfa27148440ddc3bfJan Beulich CFI_ADJUST_CFA_OFFSET -5*4 1064e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge jmp sysenter_past_esp 1065557d7d4e294ee6fb1db0cb6c1ec97a1c908b880dGlauber Costa CFI_ENDPROC 1066e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge 10675ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy FitzhardingeENTRY(xen_hypervisor_callback) 10685ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_STARTPROC 10695ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge pushl $0 10705ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET 4 10715ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge SAVE_ALL 10725ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge TRACE_IRQS_OFF 10739ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge 10749ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge /* Check to see if we got the event in the critical 10759ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge region in xen_iret_direct, after we've reenabled 10769ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge events and checked for pending events. This simulates 10779ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge iret instruction's behaviour where it delivers a 10789ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge pending interrupt when enabling interrupts. */ 10799ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge movl PT_EIP(%esp),%eax 10809ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge cmpl $xen_iret_start_crit,%eax 10819ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge jb 1f 10829ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge cmpl $xen_iret_end_crit,%eax 10839ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge jae 1f 10849ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge 10850f2c87695219b1129ccf93e0f58acdcdd49724b9Jeremy Fitzhardinge jmp xen_iret_crit_fixup 1086e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge 1087e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy FitzhardingeENTRY(xen_do_upcall) 1088b77797fb2bf31bf076e6b69736119bc6a077525bJeremy Fitzhardinge1: mov %esp, %eax 10895ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge call xen_evtchn_do_upcall 10905ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp ret_from_intr 10915ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ENDPROC 10925ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy FitzhardingeENDPROC(xen_hypervisor_callback) 10935ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge 10945ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# Hypervisor uses this for application faults while it executes. 10955ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# We get here for two reasons: 10965ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# 1. Fault while reloading DS, ES, FS or GS 10975ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# 2. Fault while executing IRET 10985ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# Category 1 we fix up by reattempting the load, and zeroing the segment 10995ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# register if the load fails. 11005ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# Category 2 we fix up by jumping to do_iret_error. We cannot use the 11015ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# normal Linux return path in this case because if we use the IRET hypercall 11025ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# to pop the stack frame we end up in an infinite loop of failsafe callbacks. 11035ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# We distinguish between categories by maintaining a status value in EAX. 11045ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy FitzhardingeENTRY(xen_failsafe_callback) 11055ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_STARTPROC 11065ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge pushl %eax 11075ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET 4 11085ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl $1,%eax 11095ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge1: mov 4(%esp),%ds 11105ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge2: mov 8(%esp),%es 11115ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge3: mov 12(%esp),%fs 11125ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge4: mov 16(%esp),%gs 11135ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge testl %eax,%eax 11145ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge popl %eax 11155ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET -4 11165ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge lea 16(%esp),%esp 11175ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET -16 11185ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jz 5f 11195ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge addl $16,%esp 11205ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp iret_exc # EAX != 0 => Category 2 (Bad IRET) 11215ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge5: pushl $0 # EAX == 0 => Category 1 (Bad segment) 11225ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET 4 11235ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge SAVE_ALL 11245ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp ret_from_exception 11255ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ENDPROC 11265ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge 11275ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge.section .fixup,"ax" 11285ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge6: xorl %eax,%eax 11295ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl %eax,4(%esp) 11305ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp 1b 11315ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge7: xorl %eax,%eax 11325ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl %eax,8(%esp) 11335ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp 2b 11345ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge8: xorl %eax,%eax 11355ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl %eax,12(%esp) 11365ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp 3b 11375ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge9: xorl %eax,%eax 11385ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl %eax,16(%esp) 11395ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp 4b 11405ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge.previous 11415ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge.section __ex_table,"a" 11425ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge .align 4 11435ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge .long 1b,6b 11445ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge .long 2b,7b 11455ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge .long 3b,8b 11465ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge .long 4b,9b 11475ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge.previous 11485ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy FitzhardingeENDPROC(xen_failsafe_callback) 11495ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge 11505ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge#endif /* CONFIG_XEN */ 11515ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge 1152606576ce816603d9fe1fb453a88bc6eea16ca709Steven Rostedt#ifdef CONFIG_FUNCTION_TRACER 1153d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt#ifdef CONFIG_DYNAMIC_FTRACE 1154d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 1155d61f82d06672f57fca410da6f7fffd15867db622Steven RostedtENTRY(mcount) 1156d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt ret 1157d61f82d06672f57fca410da6f7fffd15867db622Steven RostedtEND(mcount) 1158d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 1159d61f82d06672f57fca410da6f7fffd15867db622Steven RostedtENTRY(ftrace_caller) 116060a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt cmpl $0, function_trace_stop 116160a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt jne ftrace_stub 116260a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt 1163d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt pushl %eax 1164d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt pushl %ecx 1165d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt pushl %edx 1166d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt movl 0xc(%esp), %eax 1167d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt movl 0x4(%ebp), %edx 1168395a59d0f8e86bb39cd700c3d185d30c670bb958Abhishek Sagar subl $MCOUNT_INSN_SIZE, %eax 1169d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 1170d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt.globl ftrace_call 1171d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedtftrace_call: 1172d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt call ftrace_stub 1173d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 1174d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt popl %edx 1175d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt popl %ecx 1176d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt popl %eax 1177d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 1178d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt.globl ftrace_stub 1179d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedtftrace_stub: 1180d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt ret 1181d61f82d06672f57fca410da6f7fffd15867db622Steven RostedtEND(ftrace_caller) 1182d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 1183d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt#else /* ! CONFIG_DYNAMIC_FTRACE */ 1184d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 118516444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de MeloENTRY(mcount) 118660a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt cmpl $0, function_trace_stop 118760a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt jne ftrace_stub 118860a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt 118916444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo cmpl $ftrace_stub, ftrace_trace_function 119016444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo jnz trace 1191caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker#ifdef CONFIG_FUNCTION_RET_TRACER 1192caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker cmpl $ftrace_stub, ftrace_function_return 1193e7d3737ea1b102030f44e96c97754101e41515f0Frederic Weisbecker jnz ftrace_return_caller 1194caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker#endif 119516444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo.globl ftrace_stub 119616444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Meloftrace_stub: 119716444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo ret 119816444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo 119916444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo /* taken from glibc */ 120016444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melotrace: 120116444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo pushl %eax 120216444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo pushl %ecx 120316444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo pushl %edx 120416444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo movl 0xc(%esp), %eax 120516444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo movl 0x4(%ebp), %edx 1206395a59d0f8e86bb39cd700c3d185d30c670bb958Abhishek Sagar subl $MCOUNT_INSN_SIZE, %eax 120716444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo 1208d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt call *ftrace_trace_function 120916444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo 121016444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo popl %edx 121116444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo popl %ecx 121216444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo popl %eax 1213caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker jmp ftrace_stub 1214e7d3737ea1b102030f44e96c97754101e41515f0Frederic WeisbeckerEND(mcount) 1215e7d3737ea1b102030f44e96c97754101e41515f0Frederic Weisbecker#endif /* CONFIG_DYNAMIC_FTRACE */ 1216e7d3737ea1b102030f44e96c97754101e41515f0Frederic Weisbecker#endif /* CONFIG_FUNCTION_TRACER */ 121716444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo 1218caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker#ifdef CONFIG_FUNCTION_RET_TRACER 1219e7d3737ea1b102030f44e96c97754101e41515f0Frederic WeisbeckerENTRY(ftrace_return_caller) 1220e7d3737ea1b102030f44e96c97754101e41515f0Frederic Weisbecker cmpl $0, function_trace_stop 1221e7d3737ea1b102030f44e96c97754101e41515f0Frederic Weisbecker jne ftrace_stub 1222e7d3737ea1b102030f44e96c97754101e41515f0Frederic Weisbecker 1223caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %eax 1224caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %ecx 1225caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %edx 12261dc1c6adf38bc5799d1594681645ced40ced4b6bFrederic Weisbecker movl 0xc(%esp), %edx 1227caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker lea 0x4(%ebp), %eax 1228caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker call prepare_ftrace_return 1229caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %edx 1230caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %ecx 1231caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %eax 1232e7d3737ea1b102030f44e96c97754101e41515f0Frederic Weisbecker ret 1233e7d3737ea1b102030f44e96c97754101e41515f0Frederic WeisbeckerEND(ftrace_return_caller) 1234caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker 1235caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker.globl return_to_handler 1236caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbeckerreturn_to_handler: 1237caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl $0 1238caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %eax 1239caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %ecx 1240caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %edx 1241caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker call ftrace_return_to_handler 1242caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker movl %eax, 0xc(%esp) 1243caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %edx 1244caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %ecx 1245caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %eax 1246caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker ret 1247e7d3737ea1b102030f44e96c97754101e41515f0Frederic Weisbecker#endif 124816444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo 1249bb152f53120d66c98c1f16518407df6a84f23714Arjan van de Ven.section .rodata,"a" 1250541054d935a1ec89916977cbf0d16ddb71b3ff5cThomas Gleixner#include "syscall_table_32.S" 12511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 12521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_table_size=(.-sys_call_table) 1253