entry_32.S revision f0d96110f9fd98a1a22e03b8adba69508843d910
11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (C) 1991, 1992 Linus Torvalds 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * entry.S contains the system-call and fault low-level handling routines. 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This also contains the timer-interrupt handler, as well as all interrupts 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * and faults that can result in a task-switch. 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * NOTE: This code handles signal-recognition, which happens every time 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * after a timer-interrupt and after each system call. 131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * I changed all the .align's to 4 (16 byte alignment), as that's faster 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * on a 486. 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 17889f21ce272e38db19c8114a7e0a5793d4590077Andi Kleen * Stack layout in 'syscall_exit': 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * ptrace needs to have all regs on the stack. 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * if the order here is changed, it needs to be 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * updated in fork.c:copy_process, signal.c:do_signal, 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * ptrace.c and ptrace.h 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 0(%esp) - %ebx 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 4(%esp) - %ecx 251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 8(%esp) - %edx 261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * C(%esp) - %esi 271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 10(%esp) - %edi 281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 14(%esp) - %ebp 291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 18(%esp) - %eax 301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1C(%esp) - %ds 311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 20(%esp) - %es 32464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge * 24(%esp) - %fs 33f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge * 28(%esp) - orig_eax 34f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge * 2C(%esp) - %eip 35f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge * 30(%esp) - %cs 36f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge * 34(%esp) - %eflags 37f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge * 38(%esp) - %oldesp 38f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge * 3C(%esp) - %oldss 391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * "current" is in register %ebx during any slow entries. 411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/linkage.h> 441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/thread_info.h> 4555f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar#include <asm/irqflags.h> 461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/errno.h> 471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/segment.h> 481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/smp.h> 491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/page.h> 501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/desc.h> 51be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev#include <asm/percpu.h> 52fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich#include <asm/dwarf2.h> 53ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov#include <asm/processor-flags.h> 54395a59d0f8e86bb39cd700c3d185d30c670bb958Abhishek Sagar#include <asm/ftrace.h> 559b7dc567d03d74a1fbae84e88949b6a60d922d82Thomas Gleixner#include <asm/irq_vectors.h> 561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 57af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */ 58af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#include <linux/elf-em.h> 59af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#define AUDIT_ARCH_I386 (EM_386|__AUDIT_ARCH_LE) 60af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#define __AUDIT_ARCH_LE 0x40000000 61af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath 62af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#ifndef CONFIG_AUDITSYSCALL 63af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#define sysenter_audit syscall_trace_entry 64af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#define sysexit_audit syscall_exit_work 65af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#endif 66af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath 67139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell/* 68139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * We use macros for low-level operations which need to be overridden 69139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * for paravirtualization. The following will never clobber any registers: 70139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * INTERRUPT_RETURN (aka. "iret") 71139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") 72d75cd22fdd5f7d203fb60014d426942df33dd9a6Jeremy Fitzhardinge * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit"). 73139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * 74139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must 75139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). 76139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * Allowing a register to be clobbered can shrink the paravirt replacement 77139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * enough to patch inline, increasing performance. 78139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell */ 79139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell 801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define nr_syscalls ((syscall_table_size)/4) 811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef CONFIG_PREEMPT 83139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell#define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF 841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#else 85139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell#define preempt_stop(clobbers) 861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define resume_kernel restore_nocheck 871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8955f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar.macro TRACE_IRQS_IRET 9055f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar#ifdef CONFIG_TRACE_IRQFLAGS 91ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off? 9255f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar jz 1f 9355f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_ON 9455f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar1: 9555f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar#endif 9655f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar.endm 9755f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar 984031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov#ifdef CONFIG_VM86 994031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov#define resume_userspace_sig check_userspace 1004031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov#else 1014031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov#define resume_userspace_sig resume_userspace 1024031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov#endif 1034031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov 104f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro SAVE_ALL 105f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo cld 106f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl %fs 107f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 108f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_REL_OFFSET fs, 0;*/ 109f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl %es 110f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 111f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_REL_OFFSET es, 0;*/ 112f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl %ds 113f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 114f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_REL_OFFSET ds, 0;*/ 115f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl %eax 116f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 117f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_REL_OFFSET eax, 0 118f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl %ebp 119f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 120f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_REL_OFFSET ebp, 0 121f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl %edi 122f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 123f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_REL_OFFSET edi, 0 124f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl %esi 125f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 126f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_REL_OFFSET esi, 0 127f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl %edx 128f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 129f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_REL_OFFSET edx, 0 130f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl %ecx 131f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 132f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_REL_OFFSET ecx, 0 133f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl %ebx 134f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 135f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_REL_OFFSET ebx, 0 136f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl $(__USER_DS), %edx 137f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl %edx, %ds 138f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl %edx, %es 139f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl $(__KERNEL_PERCPU), %edx 140464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge movl %edx, %fs 141f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 1421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 143f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro RESTORE_INT_REGS 144f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo popl %ebx 145f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -4 146f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_RESTORE ebx 147f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo popl %ecx 148f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -4 149f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_RESTORE ecx 150f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo popl %edx 151f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -4 152f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_RESTORE edx 153f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo popl %esi 154f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -4 155f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_RESTORE esi 156f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo popl %edi 157f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -4 158f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_RESTORE edi 159f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo popl %ebp 160f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -4 161f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_RESTORE ebp 162f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo popl %eax 163f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -4 164fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_RESTORE eax 165f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 1661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 167f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro RESTORE_REGS 168f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo RESTORE_INT_REGS 169f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo1: popl %ds 170f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -4 171f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_RESTORE ds;*/ 172f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo2: popl %es 173f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -4 174f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_RESTORE es;*/ 175f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo3: popl %fs 176f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -4 177f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_RESTORE fs;*/ 178f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.pushsection .fixup, "ax" 179f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo4: movl $0, (%esp) 180f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo jmp 1b 181f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo5: movl $0, (%esp) 182f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo jmp 2b 183f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo6: movl $0, (%esp) 184f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo jmp 3b 185f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.section __ex_table, "a" 186f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo .align 4 187f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo .long 1b, 4b 188f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo .long 2b, 5b 189f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo .long 3b, 6b 190f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge.popsection 191f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 1921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 193f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro RING0_INT_FRAME 194f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_STARTPROC simple 195f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_SIGNAL_FRAME 196f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_DEF_CFA esp, 3*4 197f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_OFFSET cs, -2*4;*/ 198fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_OFFSET eip, -3*4 199f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 200fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich 201f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro RING0_EC_FRAME 202f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_STARTPROC simple 203f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_SIGNAL_FRAME 204f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_DEF_CFA esp, 4*4 205f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_OFFSET cs, -2*4;*/ 206fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_OFFSET eip, -3*4 207f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 208fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich 209f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro RING0_PTREGS_FRAME 210f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_STARTPROC simple 211f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_SIGNAL_FRAME 212f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_DEF_CFA esp, PT_OLDESP-PT_EBX 213f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/ 214f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_OFFSET eip, PT_EIP-PT_OLDESP 215f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/ 216f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/ 217f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_OFFSET eax, PT_EAX-PT_OLDESP 218f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_OFFSET ebp, PT_EBP-PT_OLDESP 219f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_OFFSET edi, PT_EDI-PT_OLDESP 220f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_OFFSET esi, PT_ESI-PT_OLDESP 221f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_OFFSET edx, PT_EDX-PT_OLDESP 222f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_OFFSET ecx, PT_ECX-PT_OLDESP 223eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge CFI_OFFSET ebx, PT_EBX-PT_OLDESP 224f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 2251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(ret_from_fork) 227fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_STARTPROC 2281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax 22925d7dfdaf3404bb31b8f55283fd2c456cb7b4001Markus Armbruster CFI_ADJUST_CFA_OFFSET 4 2301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call schedule_tail 2311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 2321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %eax 233fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 23447a5c6fa0e204a2b63309c648bb2fde36836c826Linus Torvalds pushl $0x0202 # Reset kernel eflags 23547a5c6fa0e204a2b63309c648bb2fde36836c826Linus Torvalds CFI_ADJUST_CFA_OFFSET 4 23647a5c6fa0e204a2b63309c648bb2fde36836c826Linus Torvalds popfl 23747a5c6fa0e204a2b63309c648bb2fde36836c826Linus Torvalds CFI_ADJUST_CFA_OFFSET -4 2381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp syscall_exit 239fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 24047a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(ret_from_fork) 2411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 2431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Return to user mode is not as complex as all this looks, 2441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * but we want the default path for a system call return to 2451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * go as quickly as possible which is why some of this is 2461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * less clear than it otherwise should be. 2471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 2481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # userspace resumption stub bypassing syscall exit tracing 2501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 251fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_PTREGS_FRAME 2521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsret_from_exception: 253139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell preempt_stop(CLBR_ANY) 2541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsret_from_intr: 2551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 2564031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelovcheck_userspace: 257eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS 258eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movb PT_CS(%esp), %al 259ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax 26078be3706b21a232310590fe00258b224177ac05fRusty Russell cmpl $USER_RPL, %eax 26178be3706b21a232310590fe00258b224177ac05fRusty Russell jb resume_kernel # not returning to v8086 or userspace 262f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge 2631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(resume_userspace) 264c7e872e7da5514d014707a407ea562d197cc0136Peter Zijlstra LOCKDEP_SYS_EXIT 265139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 2661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # setting need_resched or sigpending 2671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # between sampling and the iret 268e32e58a96de4ac35a03349db2ab69f263ded958fPeter Zijlstra TRACE_IRQS_OFF 2691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx 2701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds andl $_TIF_WORK_MASK, %ecx # is there any work to be done on 2711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # int/exception return? 2721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne work_pending 2731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp restore_all 27447a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(ret_from_exception) 2751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef CONFIG_PREEMPT 2771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(resume_kernel) 278139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_ANY) 2791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ? 2801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz restore_nocheck 2811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsneed_resched: 2821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx # need_resched set ? 2831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testb $_TIF_NEED_RESCHED, %cl 2841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz restore_all 285ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off (exception path) ? 2861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz restore_all 2871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call preempt_schedule_irq 2881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp need_resched 28947a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(resume_kernel) 2901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 291fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 2921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* SYSENTER_RETURN points to after the "sysenter" instruction in 2941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */ 2951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # sysenter call handler stub 2970aa97fb22624f18e5925d702ab0364d3838cfd91Roland McGrathENTRY(ia32_sysenter_target) 298fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_STARTPROC simple 299adf1423698f00d00b267f7dca8231340ce7d65efJan Beulich CFI_SIGNAL_FRAME 300fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_DEF_CFA esp, 0 301fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REGISTER esp, ebp 302faca62273b602ab482fb7d3d940dbf41ef08b00eH. Peter Anvin movl TSS_sysenter_sp0(%esp),%esp 3031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssysenter_past_esp: 30455f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar /* 305d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge * Interrupts are disabled here, but we can't trace it until 306d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge * enough kernel state to call TRACE_IRQS_OFF can be called - but 307d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge * we immediately enable interrupts at that point anyway. 30855f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar */ 3091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $(__USER_DS) 310fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 311fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_REL_OFFSET ss, 0*/ 3121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ebp 313fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 314fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET esp, 0 3151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushfl 316d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge orl $X86_EFLAGS_IF, (%esp) 317fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 3181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $(__USER_CS) 319fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 320fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_REL_OFFSET cs, 0*/ 321e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar /* 322e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar * Push current_thread_info()->sysenter_return to the stack. 323e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar * A tiny bit of offset fixup is necessary - 4*4 means the 4 words 324e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar * pushed above; +8 corresponds to copy_thread's esp0 setting. 325e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar */ 326e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp) 327fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 328fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET eip, 0 3291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 330d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge pushl %eax 331d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET 4 332d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge SAVE_ALL 333d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge ENABLE_INTERRUPTS(CLBR_NONE) 334d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge 3351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 3361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Load the potential sixth argument from user stack. 3371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Careful about security. 3381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 3391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $__PAGE_OFFSET-3,%ebp 3401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jae syscall_fault 3411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds1: movl (%ebp),%ebp 342d93c870bad38e8daaaf9f7e900a13431f24becbbJeremy Fitzhardinge movl %ebp,PT_EBP(%esp) 3431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.section __ex_table,"a" 3441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .align 4 3451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .long 1b,syscall_fault 3461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.previous 3471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 3491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */ 351d4d67150165df8bf1cc05e532f6efca96f907cabRoland McGrath testw $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp) 352af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jnz sysenter_audit 353af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrathsysenter_do_call: 3541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $(nr_syscalls), %eax 3551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jae syscall_badsys 3561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call *sys_call_table(,%eax,4) 357eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl %eax,PT_EAX(%esp) 358c7e872e7da5514d014707a407ea562d197cc0136Peter Zijlstra LOCKDEP_SYS_EXIT 35942c24fa22e86365055fc931d833f26165e687c19Jeremy Fitzhardinge DISABLE_INTERRUPTS(CLBR_ANY) 36055f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 3611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx 3621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testw $_TIF_ALLWORK_MASK, %cx 363af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jne sysexit_audit 364af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrathsysenter_exit: 3651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* if something modifies registers it must also disable sysexit */ 366eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_EIP(%esp), %edx 367eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_OLDESP(%esp), %ecx 3681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %ebp,%ebp 36955f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_ON 370464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge1: mov PT_FS(%esp), %fs 371d75cd22fdd5f7d203fb60014d426942df33dd9a6Jeremy Fitzhardinge ENABLE_INTERRUPTS_SYSEXIT 372af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath 373af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#ifdef CONFIG_AUDITSYSCALL 374af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrathsysenter_audit: 375af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath testw $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%ebp) 376af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jnz syscall_trace_entry 377af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath addl $4,%esp 378af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath CFI_ADJUST_CFA_OFFSET -4 379af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath /* %esi already in 8(%esp) 6th arg: 4th syscall arg */ 380af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath /* %edx already in 4(%esp) 5th arg: 3rd syscall arg */ 381af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath /* %ecx already in 0(%esp) 4th arg: 2nd syscall arg */ 382af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl %ebx,%ecx /* 3rd arg: 1st syscall arg */ 383af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl %eax,%edx /* 2nd arg: syscall number */ 384af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl $AUDIT_ARCH_I386,%eax /* 1st arg: audit arch */ 385af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath call audit_syscall_entry 386af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath pushl %ebx 387af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath CFI_ADJUST_CFA_OFFSET 4 388af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl PT_EAX(%esp),%eax /* reload syscall number */ 389af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jmp sysenter_do_call 390af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath 391af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrathsysexit_audit: 392af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath testw $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %cx 393af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jne syscall_exit_work 394af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath TRACE_IRQS_ON 395af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath ENABLE_INTERRUPTS(CLBR_ANY) 396af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl %eax,%edx /* second arg, syscall return value */ 397af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath cmpl $0,%eax /* is it < 0? */ 398af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath setl %al /* 1 if so, 0 if not */ 399af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movzbl %al,%eax /* zero-extend that */ 400af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath inc %eax /* first arg, 0->1(AUDITSC_SUCCESS), 1->2(AUDITSC_FAILURE) */ 401af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath call audit_syscall_exit 402af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath DISABLE_INTERRUPTS(CLBR_ANY) 403af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath TRACE_IRQS_OFF 404af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl TI_flags(%ebp), %ecx 405af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath testw $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %cx 406af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jne syscall_exit_work 407af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath movl PT_EAX(%esp),%eax /* reload syscall return value */ 408af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath jmp sysenter_exit 409af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath#endif 410af0575bba0f46dd9054d46e0a88c57afad3bf4d2Roland McGrath 411fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 412f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge.pushsection .fixup,"ax" 413464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge2: movl $0,PT_FS(%esp) 414f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge jmp 1b 415f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge.section __ex_table,"a" 416f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge .align 4 417f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge .long 1b,2b 418f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge.popsection 4190aa97fb22624f18e5925d702ab0364d3838cfd91Roland McGrathENDPROC(ia32_sysenter_target) 4201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # system call handler stub 4221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(system_call) 423fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME # can't unwind into user space anyway 4241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax # save orig_eax 425fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 4261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds SAVE_ALL 4271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 428ed75e8d58010fdc06e2c3a81bfbebae92314c7e3Laurent Vivier # system call tracing in operation / emulation 4291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */ 430d4d67150165df8bf1cc05e532f6efca96f907cabRoland McGrath testw $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp) 4311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz syscall_trace_entry 4321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $(nr_syscalls), %eax 4331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jae syscall_badsys 4341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_call: 4351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call *sys_call_table(,%eax,4) 436eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl %eax,PT_EAX(%esp) # store the return value 4371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_exit: 438c7e872e7da5514d014707a407ea562d197cc0136Peter Zijlstra LOCKDEP_SYS_EXIT 439139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 4401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # setting need_resched or sigpending 4411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # between sampling and the iret 44255f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 4431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx 4441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testw $_TIF_ALLWORK_MASK, %cx # current->work 4451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne syscall_exit_work 4461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsrestore_all: 448eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS 449eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge # Warning: PT_OLDSS(%esp) contains the wrong/random values if we 4505df240826c90afdc7956f55a004ea6b702df9203Stas Sergeev # are returning to the kernel. 4515df240826c90afdc7956f55a004ea6b702df9203Stas Sergeev # See comments in process.c:copy_thread() for details. 452eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movb PT_OLDSS(%esp), %ah 453eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movb PT_CS(%esp), %al 454ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax 45578be3706b21a232310590fe00258b224177ac05fRusty Russell cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax 456fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REMEMBER_STATE 4571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds je ldt_ss # returning to user-space with LDT SS 4581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsrestore_nocheck: 45955f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_IRET 46055f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnarrestore_nocheck_notrace: 4611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds RESTORE_REGS 462f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge addl $4, %esp # skip orig_eax/error_code 463fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 464f7f3d791e61d7baf8b0aee0384fdd469c0d2ac9bAdrian Bunkirq_return: 4653701d863b43d05ffeb223d269583398f914fb5d3Ingo Molnar INTERRUPT_RETURN 4661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.section .fixup,"ax" 46790e9f53662826db3cdd6d99bd394d727b05160c1Jeremy FitzhardingeENTRY(iret_exc) 468a879cbbb34cbecfa9707fbb6e5a00c503ac1ecb9Linus Torvalds pushl $0 # no error code 469a879cbbb34cbecfa9707fbb6e5a00c503ac1ecb9Linus Torvalds pushl $do_iret_error 470a879cbbb34cbecfa9707fbb6e5a00c503ac1ecb9Linus Torvalds jmp error_code 4711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.previous 4721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.section __ex_table,"a" 4731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .align 4 4743701d863b43d05ffeb223d269583398f914fb5d3Ingo Molnar .long irq_return,iret_exc 4751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.previous 4761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 477fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_RESTORE_STATE 4781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsldt_ss: 479eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge larl PT_OLDSS(%esp), %eax 4801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz restore_nocheck 4811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testl $0x00400000, %eax # returning to 32bit stack? 4821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz restore_nocheck # allright, normal return 483d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell 484d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell#ifdef CONFIG_PARAVIRT 485d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell /* 486d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * The kernel can't run on a non-flat stack if paravirt mode 487d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * is active. Rather than try to fixup the high bits of 488d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * ESP, bypass this code entirely. This may break DOSemu 489d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * and/or Wine support in a paravirt VM, although the option 490d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * is still available to implement the setting of the high 491d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * 16-bits in the INTERRUPT_RETURN paravirt-op. 492d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell */ 49393b1eab3d29e7ea32ee583de3362da84db06ded8Jeremy Fitzhardinge cmpl $0, pv_info+PARAVIRT_enabled 494d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell jne restore_nocheck 495d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell#endif 496d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell 4971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* If returning to userspace with 16bit stack, 4981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * try to fix the higher word of ESP, as the CPU 4991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * won't restore it. 5001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This is an "official" bug of all the x86-compatible 5011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * CPUs, which we can try to work around to make 5021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * dosemu and wine happy. */ 503eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_OLDESP(%esp), %eax 504be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev movl %esp, %edx 505be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev call patch_espfix_desc 506be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev pushl $__ESPFIX_SS 507be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev CFI_ADJUST_CFA_OFFSET 4 508be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev pushl %eax 509be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev CFI_ADJUST_CFA_OFFSET 4 510139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_EAX) 51155f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 512be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev lss (%esp), %esp 513be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev CFI_ADJUST_CFA_OFFSET -8 514be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev jmp restore_nocheck 515fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 51647a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichENDPROC(system_call) 5171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # perform work that needs to be done immediately before resumption 5191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 520fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_PTREGS_FRAME # can't unwind into user space anyway 5211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldswork_pending: 5221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testb $_TIF_NEED_RESCHED, %cl 5231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz work_notifysig 5241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldswork_resched: 5251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call schedule 526c7e872e7da5514d014707a407ea562d197cc0136Peter Zijlstra LOCKDEP_SYS_EXIT 527139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 5281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # setting need_resched or sigpending 5291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # between sampling and the iret 53055f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 5311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx 5321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds andl $_TIF_WORK_MASK, %ecx # is there any work to be done other 5331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # than syscall tracing? 5341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz restore_all 5351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testb $_TIF_NEED_RESCHED, %cl 5361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz work_resched 5371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldswork_notifysig: # deal with pending signals and 5391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # notify-resume requests 54074b47a7844501445d41d704fe7c626f4b1819508Joe Korty#ifdef CONFIG_VM86 541ab68ed98f665436601feec853c8f400d28c39e92Cyrill Gorcunov testl $X86_EFLAGS_VM, PT_EFLAGS(%esp) 5421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp, %eax 5431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne work_notifysig_v86 # returning to kernel-space or 5441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # vm86-space 5451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %edx, %edx 5461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_notify_resume 5474031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov jmp resume_userspace_sig 5481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 5501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldswork_notifysig_v86: 5511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ecx # save ti_flags for do_notify_resume 552fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 5531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call save_v86_state # %eax contains pt_regs pointer 5541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %ecx 555fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 5561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %eax, %esp 55774b47a7844501445d41d704fe7c626f4b1819508Joe Korty#else 55874b47a7844501445d41d704fe7c626f4b1819508Joe Korty movl %esp, %eax 55974b47a7844501445d41d704fe7c626f4b1819508Joe Korty#endif 5601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %edx, %edx 5611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_notify_resume 5624031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov jmp resume_userspace_sig 56347a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(work_pending) 5641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # perform syscall exit tracing 5661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 5671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_trace_entry: 568eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl $-ENOSYS,PT_EAX(%esp) 5691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp, %eax 570d4d67150165df8bf1cc05e532f6efca96f907cabRoland McGrath call syscall_trace_enter 571d4d67150165df8bf1cc05e532f6efca96f907cabRoland McGrath /* What it returned is what we'll actually use. */ 5721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $(nr_syscalls), %eax 5731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnae syscall_call 5741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp syscall_exit 57547a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(syscall_trace_entry) 5761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # perform syscall exit tracing 5781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 5791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_exit_work: 580d4d67150165df8bf1cc05e532f6efca96f907cabRoland McGrath testb $_TIF_WORK_SYSCALL_EXIT, %cl 5811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz work_pending 58255f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_ON 583d4d67150165df8bf1cc05e532f6efca96f907cabRoland McGrath ENABLE_INTERRUPTS(CLBR_ANY) # could let syscall_trace_leave() call 5841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # schedule() instead 5851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp, %eax 586d4d67150165df8bf1cc05e532f6efca96f907cabRoland McGrath call syscall_trace_leave 5871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp resume_userspace 58847a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(syscall_exit_work) 589fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 5901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 591fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME # can't unwind into user space anyway 5921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_fault: 5931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 594eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl $-EFAULT,PT_EAX(%esp) 5951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp resume_userspace 59647a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(syscall_fault) 5971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_badsys: 599eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl $-ENOSYS,PT_EAX(%esp) 6001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp resume_userspace 60147a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(syscall_badsys) 602fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 6031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 604f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro FIXUP_ESPFIX_STACK 605f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /* since we are on a wrong stack, we cant make it a C code :( */ 606f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo PER_CPU(gdt_page, %ebx) 607f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah) 608f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo addl %esp, %eax 609f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl $__KERNEL_DS 610f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 611f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl %eax 612f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 613f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo lss (%esp), %esp 614f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET -8 615f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 616f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro UNWIND_ESPFIX_STACK 617f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl %ss, %eax 618f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /* see if on espfix stack */ 619f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo cmpw $__ESPFIX_SS, %ax 620f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo jne 27f 621f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl $__KERNEL_DS, %eax 622f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl %eax, %ds 623f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl %eax, %es 624f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo /* switch to normal stack */ 625f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo FIXUP_ESPFIX_STACK 626f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo27: 627f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 6281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 630b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin * Build the entry stubs and pointer table with some assembler magic. 631b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin * We pack 7 stubs into a single 32-byte chunk, which will fit in a 632b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin * single cache line on all modern x86 implementations. 6331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 6344687518c4cb7807fbeff21770e309080f9eb7f2fH. Peter Anvin.section .init.rodata,"a" 6351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(interrupt) 6361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.text 637b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .p2align 5 638b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .p2align CONFIG_X86_L1_CACHE_SHIFT 6391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(irq_entries_start) 640fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 6414687518c4cb7807fbeff21770e309080f9eb7f2fH. Peter Anvinvector=FIRST_EXTERNAL_VECTOR 642b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin.rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7 643b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .balign 32 644b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .rept 7 645b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .if vector < NR_VECTORS 6468665596ec05498525014436520b316ba174a068aH. Peter Anvin .if vector <> FIRST_EXTERNAL_VECTOR 647fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 648b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .endif 649b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin1: pushl $(~vector+0x80) /* Note: always in signed byte range */ 650fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 6518665596ec05498525014436520b316ba174a068aH. Peter Anvin .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6 652b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin jmp 2f 653b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .endif 654b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .previous 6551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .long 1b 656b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .text 6571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvector=vector+1 658b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .endif 659b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .endr 660b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin2: jmp common_interrupt 6611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.endr 66247a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(irq_entries_start) 66347a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich 66447a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich.previous 66547a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(interrupt) 66647a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich.previous 6671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 66855f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar/* 66955f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar * the CPU automatically disables interrupts when executing an IRQ vector, 67055f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar * so IRQ-flags tracing has to follow that: 67155f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar */ 672b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin .p2align CONFIG_X86_L1_CACHE_SHIFT 6731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldscommon_interrupt: 674b7c6244f13d37592003b46e12500a90e9781ad9dH. Peter Anvin addl $-0x80,(%esp) /* Adjust vector into the [-256,-1] range */ 6751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds SAVE_ALL 67655f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 6771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp,%eax 6781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_IRQ 6791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp ret_from_intr 68047a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichENDPROC(common_interrupt) 681fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 6821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 68302cf94c370e0dc9bf408fe45eb86fe9ad58eaf7fTejun Heo#define BUILD_INTERRUPT3(name, nr, fn) \ 6841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(name) \ 685fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME; \ 68619eadf98c8167eac843580683317b99572e2abf0Rusty Russell pushl $~(nr); \ 687fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4; \ 688fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich SAVE_ALL; \ 68955f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF \ 6901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp,%eax; \ 69102cf94c370e0dc9bf408fe45eb86fe9ad58eaf7fTejun Heo call fn; \ 69255f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar jmp ret_from_intr; \ 69347a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich CFI_ENDPROC; \ 69447a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichENDPROC(name) 6951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 69602cf94c370e0dc9bf408fe45eb86fe9ad58eaf7fTejun Heo#define BUILD_INTERRUPT(name, nr) BUILD_INTERRUPT3(name, nr, smp_##name) 69702cf94c370e0dc9bf408fe45eb86fe9ad58eaf7fTejun Heo 6981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* The include is where all of the SMP etc. interrupts come from */ 6991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include "entry_arch.h" 7001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(coprocessor_error) 702fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 7031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 704fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_coprocessor_error 706fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 708fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 70947a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(coprocessor_error) 7101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(simd_coprocessor_error) 712fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 7131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 714fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_simd_coprocessor_error 716fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 718fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 71947a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(simd_coprocessor_error) 7201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(device_not_available) 722fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 7231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $-1 # mark this as an int 724fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7257643e9b936b4af31ba4851eb7d5b3a3bfad52502Alexander van Heukelum pushl $do_device_not_available 726fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7277643e9b936b4af31ba4851eb7d5b3a3bfad52502Alexander van Heukelum jmp error_code 728fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 72947a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(device_not_available) 7301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 731d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell#ifdef CONFIG_PARAVIRT 732d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty RussellENTRY(native_iret) 7333701d863b43d05ffeb223d269583398f914fb5d3Ingo Molnar iret 734d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell.section __ex_table,"a" 735d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell .align 4 7363701d863b43d05ffeb223d269583398f914fb5d3Ingo Molnar .long native_iret, iret_exc 737d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell.previous 73847a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(native_iret) 739d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell 740d75cd22fdd5f7d203fb60014d426942df33dd9a6Jeremy FitzhardingeENTRY(native_irq_enable_sysexit) 741d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell sti 742d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell sysexit 743d75cd22fdd5f7d203fb60014d426942df33dd9a6Jeremy FitzhardingeEND(native_irq_enable_sysexit) 744d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell#endif 745d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell 7461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(overflow) 747fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 7481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 749fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_overflow 751fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 753fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 75447a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(overflow) 7551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(bounds) 757fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 7581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 759fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_bounds 761fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 763fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 76447a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(bounds) 7651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(invalid_op) 767fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 7681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 769fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_invalid_op 771fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 773fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 77447a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(invalid_op) 7751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(coprocessor_segment_overrun) 777fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 7781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 779fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_coprocessor_segment_overrun 781fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 783fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 78447a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(coprocessor_segment_overrun) 7851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(invalid_TSS) 787fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_EC_FRAME 7881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_invalid_TSS 789fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 791fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 79247a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(invalid_TSS) 7931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(segment_not_present) 795fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_EC_FRAME 7961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_segment_not_present 797fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 799fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 80047a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(segment_not_present) 8011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(stack_segment) 803fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_EC_FRAME 8041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_stack_segment 805fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 807fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 80847a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(stack_segment) 8091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(alignment_check) 811fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_EC_FRAME 8121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_alignment_check 813fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 815fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 81647a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(alignment_check) 8171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 818d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.PENTRY(divide_error) 819d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P RING0_INT_FRAME 820d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P pushl $0 # no error code 821d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P CFI_ADJUST_CFA_OFFSET 4 822d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P pushl $do_divide_error 823fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 825fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 82647a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(divide_error) 8271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef CONFIG_X86_MCE 8291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(machine_check) 830fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 8311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 832fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 833d2f6f7aeee890df445be29a60e34925ec15f620cAndi Kleen pushl machine_check_vector 834fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 836fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 83747a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(machine_check) 8381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 8391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(spurious_interrupt_bug) 841fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 8421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 843fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_spurious_interrupt_bug 845fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 847fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 84847a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(spurious_interrupt_bug) 8491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 85002ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi KleenENTRY(kernel_thread_helper) 85102ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen pushl $0 # fake return address for unwinder 85202ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen CFI_STARTPROC 85302ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen movl %edx,%eax 85402ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen push %edx 85502ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen CFI_ADJUST_CFA_OFFSET 4 85602ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen call *%ebx 85702ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen push %eax 85802ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen CFI_ADJUST_CFA_OFFSET 4 85902ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen call do_exit 8605f5db591326779a80cfe490c5d6b6ce9fac08b31jia zhang ud2 # padding for call trace 86102ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen CFI_ENDPROC 86202ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi KleenENDPROC(kernel_thread_helper) 86302ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen 8645ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge#ifdef CONFIG_XEN 865e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge/* Xen doesn't set %esp to be precisely what the normal sysenter 866e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge entrypoint expects, so fix it up before using the normal path. */ 867e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy FitzhardingeENTRY(xen_sysenter_target) 868e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge RING0_INT_FRAME 869e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge addl $5*4, %esp /* remove xen-provided frame */ 8702ddf9b7b3e6660199269e34cfa27148440ddc3bfJan Beulich CFI_ADJUST_CFA_OFFSET -5*4 871e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge jmp sysenter_past_esp 872557d7d4e294ee6fb1db0cb6c1ec97a1c908b880dGlauber Costa CFI_ENDPROC 873e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge 8745ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy FitzhardingeENTRY(xen_hypervisor_callback) 8755ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_STARTPROC 8765ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge pushl $0 8775ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET 4 8785ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge SAVE_ALL 8795ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge TRACE_IRQS_OFF 8809ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge 8819ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge /* Check to see if we got the event in the critical 8829ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge region in xen_iret_direct, after we've reenabled 8839ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge events and checked for pending events. This simulates 8849ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge iret instruction's behaviour where it delivers a 8859ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge pending interrupt when enabling interrupts. */ 8869ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge movl PT_EIP(%esp),%eax 8879ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge cmpl $xen_iret_start_crit,%eax 8889ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge jb 1f 8899ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge cmpl $xen_iret_end_crit,%eax 8909ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge jae 1f 8919ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge 8920f2c87695219b1129ccf93e0f58acdcdd49724b9Jeremy Fitzhardinge jmp xen_iret_crit_fixup 893e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy Fitzhardinge 894e2a81baf6604a2e08e10c7405b0349106f77c8afJeremy FitzhardingeENTRY(xen_do_upcall) 895b77797fb2bf31bf076e6b69736119bc6a077525bJeremy Fitzhardinge1: mov %esp, %eax 8965ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge call xen_evtchn_do_upcall 8975ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp ret_from_intr 8985ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ENDPROC 8995ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy FitzhardingeENDPROC(xen_hypervisor_callback) 9005ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge 9015ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# Hypervisor uses this for application faults while it executes. 9025ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# We get here for two reasons: 9035ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# 1. Fault while reloading DS, ES, FS or GS 9045ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# 2. Fault while executing IRET 9055ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# Category 1 we fix up by reattempting the load, and zeroing the segment 9065ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# register if the load fails. 9075ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# Category 2 we fix up by jumping to do_iret_error. We cannot use the 9085ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# normal Linux return path in this case because if we use the IRET hypercall 9095ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# to pop the stack frame we end up in an infinite loop of failsafe callbacks. 9105ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# We distinguish between categories by maintaining a status value in EAX. 9115ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy FitzhardingeENTRY(xen_failsafe_callback) 9125ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_STARTPROC 9135ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge pushl %eax 9145ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET 4 9155ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl $1,%eax 9165ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge1: mov 4(%esp),%ds 9175ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge2: mov 8(%esp),%es 9185ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge3: mov 12(%esp),%fs 9195ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge4: mov 16(%esp),%gs 9205ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge testl %eax,%eax 9215ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge popl %eax 9225ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET -4 9235ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge lea 16(%esp),%esp 9245ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET -16 9255ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jz 5f 9265ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge addl $16,%esp 9275ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp iret_exc # EAX != 0 => Category 2 (Bad IRET) 9285ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge5: pushl $0 # EAX == 0 => Category 1 (Bad segment) 9295ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET 4 9305ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge SAVE_ALL 9315ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp ret_from_exception 9325ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ENDPROC 9335ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge 9345ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge.section .fixup,"ax" 9355ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge6: xorl %eax,%eax 9365ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl %eax,4(%esp) 9375ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp 1b 9385ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge7: xorl %eax,%eax 9395ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl %eax,8(%esp) 9405ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp 2b 9415ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge8: xorl %eax,%eax 9425ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl %eax,12(%esp) 9435ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp 3b 9445ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge9: xorl %eax,%eax 9455ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl %eax,16(%esp) 9465ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp 4b 9475ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge.previous 9485ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge.section __ex_table,"a" 9495ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge .align 4 9505ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge .long 1b,6b 9515ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge .long 2b,7b 9525ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge .long 3b,8b 9535ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge .long 4b,9b 9545ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge.previous 9555ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy FitzhardingeENDPROC(xen_failsafe_callback) 9565ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge 9575ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge#endif /* CONFIG_XEN */ 9585ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge 959606576ce816603d9fe1fb453a88bc6eea16ca709Steven Rostedt#ifdef CONFIG_FUNCTION_TRACER 960d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt#ifdef CONFIG_DYNAMIC_FTRACE 961d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 962d61f82d06672f57fca410da6f7fffd15867db622Steven RostedtENTRY(mcount) 963d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt ret 964d61f82d06672f57fca410da6f7fffd15867db622Steven RostedtEND(mcount) 965d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 966d61f82d06672f57fca410da6f7fffd15867db622Steven RostedtENTRY(ftrace_caller) 96760a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt cmpl $0, function_trace_stop 96860a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt jne ftrace_stub 96960a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt 970d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt pushl %eax 971d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt pushl %ecx 972d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt pushl %edx 973d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt movl 0xc(%esp), %eax 974d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt movl 0x4(%ebp), %edx 975395a59d0f8e86bb39cd700c3d185d30c670bb958Abhishek Sagar subl $MCOUNT_INSN_SIZE, %eax 976d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 977d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt.globl ftrace_call 978d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedtftrace_call: 979d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt call ftrace_stub 980d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 981d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt popl %edx 982d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt popl %ecx 983d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt popl %eax 9845a45cfe1c64862e8cd3b0d79d7c4ba71c3118915Steven Rostedt#ifdef CONFIG_FUNCTION_GRAPH_TRACER 9855a45cfe1c64862e8cd3b0d79d7c4ba71c3118915Steven Rostedt.globl ftrace_graph_call 9865a45cfe1c64862e8cd3b0d79d7c4ba71c3118915Steven Rostedtftrace_graph_call: 9875a45cfe1c64862e8cd3b0d79d7c4ba71c3118915Steven Rostedt jmp ftrace_stub 9885a45cfe1c64862e8cd3b0d79d7c4ba71c3118915Steven Rostedt#endif 989d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 990d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt.globl ftrace_stub 991d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedtftrace_stub: 992d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt ret 993d61f82d06672f57fca410da6f7fffd15867db622Steven RostedtEND(ftrace_caller) 994d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 995d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt#else /* ! CONFIG_DYNAMIC_FTRACE */ 996d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt 99716444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de MeloENTRY(mcount) 99860a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt cmpl $0, function_trace_stop 99960a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt jne ftrace_stub 100060a7ecf42661f2b22168751298592da6ee210c9eSteven Rostedt 100116444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo cmpl $ftrace_stub, ftrace_trace_function 100216444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo jnz trace 1003fb52607afcd0629776f1dc9e657647ceae81dd50Frederic Weisbecker#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1004c2324b694fa8ffee382a124198c68754088e483cIngo Molnar cmpl $ftrace_stub, ftrace_graph_return 1005fb52607afcd0629776f1dc9e657647ceae81dd50Frederic Weisbecker jnz ftrace_graph_caller 1006e49dc19c6a19ea112fcb94b7c62ec62cdd5c08aaSteven Rostedt 1007e49dc19c6a19ea112fcb94b7c62ec62cdd5c08aaSteven Rostedt cmpl $ftrace_graph_entry_stub, ftrace_graph_entry 1008e49dc19c6a19ea112fcb94b7c62ec62cdd5c08aaSteven Rostedt jnz ftrace_graph_caller 1009caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker#endif 101016444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo.globl ftrace_stub 101116444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Meloftrace_stub: 101216444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo ret 101316444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo 101416444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo /* taken from glibc */ 101516444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melotrace: 101616444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo pushl %eax 101716444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo pushl %ecx 101816444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo pushl %edx 101916444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo movl 0xc(%esp), %eax 102016444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo movl 0x4(%ebp), %edx 1021395a59d0f8e86bb39cd700c3d185d30c670bb958Abhishek Sagar subl $MCOUNT_INSN_SIZE, %eax 102216444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo 1023d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt call *ftrace_trace_function 102416444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo 102516444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo popl %edx 102616444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo popl %ecx 102716444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo popl %eax 102816444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo jmp ftrace_stub 102916444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de MeloEND(mcount) 1030d61f82d06672f57fca410da6f7fffd15867db622Steven Rostedt#endif /* CONFIG_DYNAMIC_FTRACE */ 1031606576ce816603d9fe1fb453a88bc6eea16ca709Steven Rostedt#endif /* CONFIG_FUNCTION_TRACER */ 103216444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo 1033fb52607afcd0629776f1dc9e657647ceae81dd50Frederic Weisbecker#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1034fb52607afcd0629776f1dc9e657647ceae81dd50Frederic WeisbeckerENTRY(ftrace_graph_caller) 1035e7d3737ea1b102030f44e96c97754101e41515f0Frederic Weisbecker cmpl $0, function_trace_stop 1036e7d3737ea1b102030f44e96c97754101e41515f0Frederic Weisbecker jne ftrace_stub 1037e7d3737ea1b102030f44e96c97754101e41515f0Frederic Weisbecker 1038caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %eax 1039caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %ecx 1040caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %edx 10411dc1c6adf38bc5799d1594681645ced40ced4b6bFrederic Weisbecker movl 0xc(%esp), %edx 1042caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker lea 0x4(%ebp), %eax 1043bb4304c71c97bf727ec43cd2f195c2c237c27fd3Steven Rostedt subl $MCOUNT_INSN_SIZE, %edx 1044caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker call prepare_ftrace_return 1045caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %edx 1046caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %ecx 1047caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %eax 1048e7d3737ea1b102030f44e96c97754101e41515f0Frederic Weisbecker ret 1049fb52607afcd0629776f1dc9e657647ceae81dd50Frederic WeisbeckerEND(ftrace_graph_caller) 1050caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker 1051caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker.globl return_to_handler 1052caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbeckerreturn_to_handler: 1053caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl $0 1054caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %eax 1055caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %ecx 1056caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker pushl %edx 1057caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker call ftrace_return_to_handler 1058caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker movl %eax, 0xc(%esp) 1059caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %edx 1060caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %ecx 1061caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker popl %eax 1062caf4b323b02a16c92fba449952ac6515ddc76d7aFrederic Weisbecker ret 1063e7d3737ea1b102030f44e96c97754101e41515f0Frederic Weisbecker#endif 106416444a8a40d4c7b4f6de34af0cae1f76a4f6c901Arnaldo Carvalho de Melo 1065bb152f53120d66c98c1f16518407df6a84f23714Arjan van de Ven.section .rodata,"a" 1066541054d935a1ec89916977cbf0d16ddb71b3ff5cThomas Gleixner#include "syscall_table_32.S" 10671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 10681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_table_size=(.-sys_call_table) 1069d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1070d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum/* 1071d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * Some functions should be protected against kprobes 1072d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum */ 1073d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum .pushsection .kprobes.text, "ax" 1074d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1075d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumENTRY(page_fault) 1076d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum RING0_EC_FRAME 1077d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl $do_page_fault 1078d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1079d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum ALIGN 1080d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelumerror_code: 1081d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /* the function address is in %fs's slot on the stack */ 1082d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %es 1083d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1084d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /*CFI_REL_OFFSET es, 0*/ 1085d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %ds 1086d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1087d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /*CFI_REL_OFFSET ds, 0*/ 1088d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %eax 1089d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1090d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET eax, 0 1091d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %ebp 1092d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1093d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET ebp, 0 1094d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %edi 1095d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1096d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET edi, 0 1097d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %esi 1098d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1099d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET esi, 0 1100d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %edx 1101d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1102d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET edx, 0 1103d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %ecx 1104d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1105d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET ecx, 0 1106d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %ebx 1107d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1108d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET ebx, 0 1109d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cld 1110d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %fs 1111d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1112d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /*CFI_REL_OFFSET fs, 0*/ 1113d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl $(__KERNEL_PERCPU), %ecx 1114d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %ecx, %fs 1115d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum UNWIND_ESPFIX_STACK 1116d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum popl %ecx 1117d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET -4 1118d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /*CFI_REGISTER es, ecx*/ 1119d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl PT_FS(%esp), %edi # get the function address 1120d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl PT_ORIG_EAX(%esp), %edx # get the error code 1121d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart 1122d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum mov %ecx, PT_FS(%esp) 1123d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /*CFI_REL_OFFSET fs, ES*/ 1124d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl $(__USER_DS), %ecx 1125d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %ecx, %ds 1126d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %ecx, %es 1127d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum TRACE_IRQS_OFF 1128d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %esp,%eax # pt_regs pointer 1129d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum call *%edi 1130d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp ret_from_exception 1131d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ENDPROC 1132d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumEND(page_fault) 1133d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1134d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum/* 1135d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * Debug traps and NMI can happen at the one SYSENTER instruction 1136d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * that sets up the real kernel stack. Check here, since we can't 1137d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * allow the wrong stack to be used. 1138d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * 1139d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have 1140d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * already pushed 3 words if it hits on the sysenter instruction: 1141d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * eflags, cs and eip. 1142d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * 1143d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * We just load the right stack, and push the three (known) values 1144d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * by hand onto the new stack - while updating the return eip past 1145d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * the instruction that would have done it for sysenter. 1146d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum */ 1147f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.macro FIX_STACK offset ok label 1148f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo cmpw $__KERNEL_CS, 4(%esp) 1149f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo jne \ok 1150f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo\label: 1151f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo movl TSS_sysenter_sp0 + \offset(%esp), %esp 1152f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_DEF_CFA esp, 0 1153f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_UNDEFINED eip 1154f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushfl 1155f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 1156f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl $__KERNEL_CS 1157f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 1158f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo pushl $sysenter_past_esp 1159f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo CFI_ADJUST_CFA_OFFSET 4 1160d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_REL_OFFSET eip, 0 1161f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo.endm 1162d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1163d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumENTRY(debug) 1164d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum RING0_INT_FRAME 1165d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpl $ia32_sysenter_target,(%esp) 1166d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jne debug_stack_correct 1167f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn 1168d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelumdebug_stack_correct: 1169d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl $-1 # mark this as an int 1170d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1171d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum SAVE_ALL 1172d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum TRACE_IRQS_OFF 1173d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum xorl %edx,%edx # error code 0 1174d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %esp,%eax # pt_regs pointer 1175d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum call do_debug 1176d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp ret_from_exception 1177d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ENDPROC 1178d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumEND(debug) 1179d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1180d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum/* 1181d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * NMI is doubly nasty. It can happen _while_ we're handling 1182d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * a debug fault, and the debug fault hasn't yet been able to 1183d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * clear up the stack. So we first check whether we got an 1184d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * NMI on the sysenter entry path, but after that we need to 1185d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * check whether we got an NMI on the debug path where the debug 1186d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * fault happened on the sysenter path. 1187d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum */ 1188d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumENTRY(nmi) 1189d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum RING0_INT_FRAME 1190d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %eax 1191d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1192d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %ss, %eax 1193d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpw $__ESPFIX_SS, %ax 1194d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum popl %eax 1195d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET -4 1196d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum je nmi_espfix_stack 1197d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpl $ia32_sysenter_target,(%esp) 1198d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum je nmi_stack_fixup 1199d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %eax 1200d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1201d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %esp,%eax 1202d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /* Do not access memory above the end of our stack page, 1203d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * it might not exist. 1204d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum */ 1205d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum andl $(THREAD_SIZE-1),%eax 1206d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpl $(THREAD_SIZE-20),%eax 1207d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum popl %eax 1208d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET -4 1209d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jae nmi_stack_correct 1210d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpl $ia32_sysenter_target,12(%esp) 1211d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum je nmi_debug_stack_check 1212d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelumnmi_stack_correct: 1213d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /* We have a RING0_INT_FRAME here */ 1214d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %eax 1215d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1216d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum SAVE_ALL 1217d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum xorl %edx,%edx # zero error code 1218d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %esp,%eax # pt_regs pointer 1219d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum call do_nmi 1220d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp restore_nocheck_notrace 1221d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ENDPROC 1222d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1223d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelumnmi_stack_fixup: 1224d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum RING0_INT_FRAME 1225f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo FIX_STACK 12, nmi_stack_correct, 1 1226d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp nmi_stack_correct 1227d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1228d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelumnmi_debug_stack_check: 1229d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /* We have a RING0_INT_FRAME here */ 1230d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpw $__KERNEL_CS,16(%esp) 1231d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jne nmi_stack_correct 1232d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpl $debug,(%esp) 1233d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jb nmi_stack_correct 1234d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum cmpl $debug_esp_fix_insn,(%esp) 1235d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum ja nmi_stack_correct 1236f0d96110f9fd98a1a22e03b8adba69508843d910Tejun Heo FIX_STACK 24, nmi_stack_correct, 1 1237d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp nmi_stack_correct 1238d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1239d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelumnmi_espfix_stack: 1240d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /* We have a RING0_INT_FRAME here. 1241d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * 1242d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * create the pointer to lss back 1243d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum */ 1244d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %ss 1245d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1246d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %esp 1247d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1248d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum addw $4, (%esp) 1249d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum /* copy the iret frame of 12 bytes */ 1250d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum .rept 3 1251d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl 16(%esp) 1252d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1253d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum .endr 1254d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl %eax 1255d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1256d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum SAVE_ALL 1257d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum FIXUP_ESPFIX_STACK # %eax == %esp 1258d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum xorl %edx,%edx # zero error code 1259d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum call do_nmi 1260d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum RESTORE_REGS 1261d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum lss 12+4(%esp), %esp # back to espfix stack 1262d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET -24 1263d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp irq_return 1264d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ENDPROC 1265d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumEND(nmi) 1266d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1267d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumENTRY(int3) 1268d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum RING0_INT_FRAME 1269d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl $-1 # mark this as an int 1270d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1271d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum SAVE_ALL 1272d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum TRACE_IRQS_OFF 1273d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum xorl %edx,%edx # zero error code 1274d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum movl %esp,%eax # pt_regs pointer 1275d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum call do_int3 1276d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp ret_from_exception 1277d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ENDPROC 1278d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumEND(int3) 1279d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1280d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumENTRY(general_protection) 1281d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum RING0_EC_FRAME 1282d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum pushl $do_general_protection 1283d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ADJUST_CFA_OFFSET 4 1284d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum jmp error_code 1285d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum CFI_ENDPROC 1286d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van HeukelumEND(general_protection) 1287d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum 1288d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum/* 1289d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum * End of kprobes section 1290d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum */ 1291d211af055d0c12dc3416c2886e6fbdc6eb74a381Alexander van Heukelum .popsection 1292