entry_32.S revision faca62273b602ab482fb7d3d940dbf41ef08b00e
11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * linux/arch/i386/entry.S 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (C) 1991, 1992 Linus Torvalds 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * entry.S contains the system-call and fault low-level handling routines. 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This also contains the timer-interrupt handler, as well as all interrupts 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * and faults that can result in a task-switch. 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * NOTE: This code handles signal-recognition, which happens every time 131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * after a timer-interrupt and after each system call. 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * I changed all the .align's to 4 (16 byte alignment), as that's faster 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * on a 486. 171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 18889f21ce272e38db19c8114a7e0a5793d4590077Andi Kleen * Stack layout in 'syscall_exit': 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * ptrace needs to have all regs on the stack. 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * if the order here is changed, it needs to be 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * updated in fork.c:copy_process, signal.c:do_signal, 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * ptrace.c and ptrace.h 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 0(%esp) - %ebx 251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 4(%esp) - %ecx 261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 8(%esp) - %edx 271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * C(%esp) - %esi 281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 10(%esp) - %edi 291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 14(%esp) - %ebp 301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 18(%esp) - %eax 311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1C(%esp) - %ds 321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 20(%esp) - %es 33464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge * 24(%esp) - %fs 34f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge * 28(%esp) - orig_eax 35f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge * 2C(%esp) - %eip 36f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge * 30(%esp) - %cs 37f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge * 34(%esp) - %eflags 38f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge * 38(%esp) - %oldesp 39f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge * 3C(%esp) - %oldss 401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * "current" is in register %ebx during any slow entries. 421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/linkage.h> 451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/thread_info.h> 4655f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar#include <asm/irqflags.h> 471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/errno.h> 481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/segment.h> 491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/smp.h> 501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/page.h> 511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/desc.h> 52be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev#include <asm/percpu.h> 53fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich#include <asm/dwarf2.h> 541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include "irq_vectors.h" 551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 56139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell/* 57139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * We use macros for low-level operations which need to be overridden 58139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * for paravirtualization. The following will never clobber any registers: 59139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * INTERRUPT_RETURN (aka. "iret") 60139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") 616abcd98ffafbff81f0bfd7ee1d129e634af13245Glauber de Oliveira Costa * ENABLE_INTERRUPTS_SYSCALL_RET (aka "sti; sysexit"). 62139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * 63139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must 64139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). 65139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * Allowing a register to be clobbered can shrink the paravirt replacement 66139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell * enough to patch inline, increasing performance. 67139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell */ 68139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell 691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define nr_syscalls ((syscall_table_size)/4) 701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsCF_MASK = 0x00000001 721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsTF_MASK = 0x00000100 731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsIF_MASK = 0x00000200 741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsDF_MASK = 0x00000400 751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsNT_MASK = 0x00004000 761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsVM_MASK = 0x00020000 771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef CONFIG_PREEMPT 79139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell#define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF 801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#else 81139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell#define preempt_stop(clobbers) 821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define resume_kernel restore_nocheck 831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8555f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar.macro TRACE_IRQS_IRET 8655f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar#ifdef CONFIG_TRACE_IRQFLAGS 87eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge testl $IF_MASK,PT_EFLAGS(%esp) # interrupts off? 8855f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar jz 1f 8955f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_ON 9055f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar1: 9155f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar#endif 9255f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar.endm 9355f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar 944031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov#ifdef CONFIG_VM86 954031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov#define resume_userspace_sig check_userspace 964031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov#else 974031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov#define resume_userspace_sig resume_userspace 984031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov#endif 994031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov 1001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define SAVE_ALL \ 1011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cld; \ 102464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge pushl %fs; \ 103f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET 4;\ 104464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge /*CFI_REL_OFFSET fs, 0;*/\ 1051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %es; \ 106fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4;\ 107fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_REL_OFFSET es, 0;*/\ 1081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ds; \ 109fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4;\ 110fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_REL_OFFSET ds, 0;*/\ 1111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax; \ 112fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4;\ 113fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET eax, 0;\ 1141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ebp; \ 115fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4;\ 116fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET ebp, 0;\ 1171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %edi; \ 118fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4;\ 119fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET edi, 0;\ 1201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %esi; \ 121fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4;\ 122fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET esi, 0;\ 1231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %edx; \ 124fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4;\ 125fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET edx, 0;\ 1261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ecx; \ 127fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4;\ 128fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET ecx, 0;\ 1291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ebx; \ 130fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4;\ 131fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET ebx, 0;\ 1321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl $(__USER_DS), %edx; \ 1331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %edx, %ds; \ 134f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge movl %edx, %es; \ 1357c3576d261ce046789a7db14f43303f8120910c7Jeremy Fitzhardinge movl $(__KERNEL_PERCPU), %edx; \ 136464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge movl %edx, %fs 1371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define RESTORE_INT_REGS \ 1391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %ebx; \ 140fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4;\ 141fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_RESTORE ebx;\ 1421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %ecx; \ 143fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4;\ 144fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_RESTORE ecx;\ 1451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %edx; \ 146fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4;\ 147fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_RESTORE edx;\ 1481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %esi; \ 149fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4;\ 150fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_RESTORE esi;\ 1511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %edi; \ 152fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4;\ 153fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_RESTORE edi;\ 1541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %ebp; \ 155fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4;\ 156fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_RESTORE ebp;\ 157fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich popl %eax; \ 158fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4;\ 159fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_RESTORE eax 1601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define RESTORE_REGS \ 1621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds RESTORE_INT_REGS; \ 1631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds1: popl %ds; \ 164fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4;\ 165fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_RESTORE ds;*/\ 1661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds2: popl %es; \ 167fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4;\ 168fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_RESTORE es;*/\ 169464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge3: popl %fs; \ 170f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET -4;\ 171464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge /*CFI_RESTORE fs;*/\ 172f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge.pushsection .fixup,"ax"; \ 1731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds4: movl $0,(%esp); \ 174f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge jmp 1b; \ 175f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge5: movl $0,(%esp); \ 1761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp 2b; \ 177f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge6: movl $0,(%esp); \ 178f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge jmp 3b; \ 1791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.section __ex_table,"a";\ 1801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .align 4; \ 181f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge .long 1b,4b; \ 182f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge .long 2b,5b; \ 183f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge .long 3b,6b; \ 184f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge.popsection 1851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 186fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich#define RING0_INT_FRAME \ 187fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_STARTPROC simple;\ 188adf1423698f00d00b267f7dca8231340ce7d65efJan Beulich CFI_SIGNAL_FRAME;\ 189fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_DEF_CFA esp, 3*4;\ 190fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_OFFSET cs, -2*4;*/\ 191fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_OFFSET eip, -3*4 192fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich 193fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich#define RING0_EC_FRAME \ 194fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_STARTPROC simple;\ 195adf1423698f00d00b267f7dca8231340ce7d65efJan Beulich CFI_SIGNAL_FRAME;\ 196fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_DEF_CFA esp, 4*4;\ 197fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_OFFSET cs, -2*4;*/\ 198fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_OFFSET eip, -3*4 199fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich 200fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich#define RING0_PTREGS_FRAME \ 201fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_STARTPROC simple;\ 202adf1423698f00d00b267f7dca8231340ce7d65efJan Beulich CFI_SIGNAL_FRAME;\ 203eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge CFI_DEF_CFA esp, PT_OLDESP-PT_EBX;\ 204eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/\ 205eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge CFI_OFFSET eip, PT_EIP-PT_OLDESP;\ 206eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/\ 207eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/\ 208eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge CFI_OFFSET eax, PT_EAX-PT_OLDESP;\ 209eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge CFI_OFFSET ebp, PT_EBP-PT_OLDESP;\ 210eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge CFI_OFFSET edi, PT_EDI-PT_OLDESP;\ 211eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge CFI_OFFSET esi, PT_ESI-PT_OLDESP;\ 212eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge CFI_OFFSET edx, PT_EDX-PT_OLDESP;\ 213eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge CFI_OFFSET ecx, PT_ECX-PT_OLDESP;\ 214eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge CFI_OFFSET ebx, PT_EBX-PT_OLDESP 2151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(ret_from_fork) 217fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_STARTPROC 2181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax 21925d7dfdaf3404bb31b8f55283fd2c456cb7b4001Markus Armbruster CFI_ADJUST_CFA_OFFSET 4 2201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call schedule_tail 2211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 2221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %eax 223fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 22447a5c6fa0e204a2b63309c648bb2fde36836c826Linus Torvalds pushl $0x0202 # Reset kernel eflags 22547a5c6fa0e204a2b63309c648bb2fde36836c826Linus Torvalds CFI_ADJUST_CFA_OFFSET 4 22647a5c6fa0e204a2b63309c648bb2fde36836c826Linus Torvalds popfl 22747a5c6fa0e204a2b63309c648bb2fde36836c826Linus Torvalds CFI_ADJUST_CFA_OFFSET -4 2281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp syscall_exit 229fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 23047a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(ret_from_fork) 2311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 2331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Return to user mode is not as complex as all this looks, 2341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * but we want the default path for a system call return to 2351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * go as quickly as possible which is why some of this is 2361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * less clear than it otherwise should be. 2371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 2381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # userspace resumption stub bypassing syscall exit tracing 2401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 241fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_PTREGS_FRAME 2421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsret_from_exception: 243139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell preempt_stop(CLBR_ANY) 2441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsret_from_intr: 2451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 2464031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelovcheck_userspace: 247eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS 248eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movb PT_CS(%esp), %al 24978be3706b21a232310590fe00258b224177ac05fRusty Russell andl $(VM_MASK | SEGMENT_RPL_MASK), %eax 25078be3706b21a232310590fe00258b224177ac05fRusty Russell cmpl $USER_RPL, %eax 25178be3706b21a232310590fe00258b224177ac05fRusty Russell jb resume_kernel # not returning to v8086 or userspace 252f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge 2531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(resume_userspace) 254c7e872e7da5514d014707a407ea562d197cc0136Peter Zijlstra LOCKDEP_SYS_EXIT 255139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 2561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # setting need_resched or sigpending 2571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # between sampling and the iret 2581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx 2591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds andl $_TIF_WORK_MASK, %ecx # is there any work to be done on 2601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # int/exception return? 2611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne work_pending 2621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp restore_all 26347a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(ret_from_exception) 2641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef CONFIG_PREEMPT 2661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(resume_kernel) 267139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_ANY) 2681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ? 2691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz restore_nocheck 2701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsneed_resched: 2711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx # need_resched set ? 2721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testb $_TIF_NEED_RESCHED, %cl 2731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz restore_all 274eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge testl $IF_MASK,PT_EFLAGS(%esp) # interrupts off (exception path) ? 2751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz restore_all 2761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call preempt_schedule_irq 2771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp need_resched 27847a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(resume_kernel) 2791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 280fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 2811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* SYSENTER_RETURN points to after the "sysenter" instruction in 2831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */ 2841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # sysenter call handler stub 2860aa97fb22624f18e5925d702ab0364d3838cfd91Roland McGrathENTRY(ia32_sysenter_target) 287fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_STARTPROC simple 288adf1423698f00d00b267f7dca8231340ce7d65efJan Beulich CFI_SIGNAL_FRAME 289fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_DEF_CFA esp, 0 290fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REGISTER esp, ebp 291faca62273b602ab482fb7d3d940dbf41ef08b00eH. Peter Anvin movl TSS_sysenter_sp0(%esp),%esp 2921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssysenter_past_esp: 29355f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar /* 29455f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar * No need to follow this irqs on/off section: the syscall 29555f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar * disabled irqs and here we enable it straight after entry: 29655f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar */ 297139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell ENABLE_INTERRUPTS(CLBR_NONE) 2981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $(__USER_DS) 299fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 300fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_REL_OFFSET ss, 0*/ 3011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ebp 302fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 303fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET esp, 0 3041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushfl 305fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 3061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $(__USER_CS) 307fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 308fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_REL_OFFSET cs, 0*/ 309e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar /* 310e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar * Push current_thread_info()->sysenter_return to the stack. 311e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar * A tiny bit of offset fixup is necessary - 4*4 means the 4 words 312e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar * pushed above; +8 corresponds to copy_thread's esp0 setting. 313e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar */ 314e6e5494cb23d1933735ee47cc674ffe1c4afed6fIngo Molnar pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp) 315fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 316fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET eip, 0 3171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 3191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Load the potential sixth argument from user stack. 3201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Careful about security. 3211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 3221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $__PAGE_OFFSET-3,%ebp 3231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jae syscall_fault 3241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds1: movl (%ebp),%ebp 3251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.section __ex_table,"a" 3261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .align 4 3271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .long 1b,syscall_fault 3281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.previous 3291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax 331fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 3321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds SAVE_ALL 3331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 3341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */ 336ed75e8d58010fdc06e2c3a81bfbebae92314c7e3Laurent Vivier testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp) 3371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz syscall_trace_entry 3381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $(nr_syscalls), %eax 3391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jae syscall_badsys 3401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call *sys_call_table(,%eax,4) 341eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl %eax,PT_EAX(%esp) 342c7e872e7da5514d014707a407ea562d197cc0136Peter Zijlstra LOCKDEP_SYS_EXIT 34342c24fa22e86365055fc931d833f26165e687c19Jeremy Fitzhardinge DISABLE_INTERRUPTS(CLBR_ANY) 34455f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 3451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx 3461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testw $_TIF_ALLWORK_MASK, %cx 3471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne syscall_exit_work 3481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* if something modifies registers it must also disable sysexit */ 349eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_EIP(%esp), %edx 350eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_OLDESP(%esp), %ecx 3511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %ebp,%ebp 35255f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_ON 353464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge1: mov PT_FS(%esp), %fs 3546abcd98ffafbff81f0bfd7ee1d129e634af13245Glauber de Oliveira Costa ENABLE_INTERRUPTS_SYSCALL_RET 355fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 356f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge.pushsection .fixup,"ax" 357464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge2: movl $0,PT_FS(%esp) 358f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge jmp 1b 359f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge.section __ex_table,"a" 360f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge .align 4 361f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge .long 1b,2b 362f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge.popsection 3630aa97fb22624f18e5925d702ab0364d3838cfd91Roland McGrathENDPROC(ia32_sysenter_target) 3641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # system call handler stub 3661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(system_call) 367fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME # can't unwind into user space anyway 3681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax # save orig_eax 369fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 3701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds SAVE_ALL 3711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 372ed75e8d58010fdc06e2c3a81bfbebae92314c7e3Laurent Vivier # system call tracing in operation / emulation 3731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */ 374ed75e8d58010fdc06e2c3a81bfbebae92314c7e3Laurent Vivier testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp) 3751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz syscall_trace_entry 3761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $(nr_syscalls), %eax 3771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jae syscall_badsys 3781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_call: 3791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call *sys_call_table(,%eax,4) 380eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl %eax,PT_EAX(%esp) # store the return value 3811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_exit: 382c7e872e7da5514d014707a407ea562d197cc0136Peter Zijlstra LOCKDEP_SYS_EXIT 383139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 3841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # setting need_resched or sigpending 3851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # between sampling and the iret 38655f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 3871e2e99f0e4aa6363e8515ed17011c210c8f1b52aJason Wessel testl $TF_MASK,PT_EFLAGS(%esp) # If tracing set singlestep flag on exit 3881e2e99f0e4aa6363e8515ed17011c210c8f1b52aJason Wessel jz no_singlestep 3891e2e99f0e4aa6363e8515ed17011c210c8f1b52aJason Wessel orl $_TIF_SINGLESTEP,TI_flags(%ebp) 3901e2e99f0e4aa6363e8515ed17011c210c8f1b52aJason Wesselno_singlestep: 3911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx 3921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testw $_TIF_ALLWORK_MASK, %cx # current->work 3931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne syscall_exit_work 3941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsrestore_all: 396eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS 397eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge # Warning: PT_OLDSS(%esp) contains the wrong/random values if we 3985df240826c90afdc7956f55a004ea6b702df9203Stas Sergeev # are returning to the kernel. 3995df240826c90afdc7956f55a004ea6b702df9203Stas Sergeev # See comments in process.c:copy_thread() for details. 400eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movb PT_OLDSS(%esp), %ah 401eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movb PT_CS(%esp), %al 40278be3706b21a232310590fe00258b224177ac05fRusty Russell andl $(VM_MASK | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax 40378be3706b21a232310590fe00258b224177ac05fRusty Russell cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax 404fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REMEMBER_STATE 4051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds je ldt_ss # returning to user-space with LDT SS 4061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsrestore_nocheck: 40755f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_IRET 40855f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnarrestore_nocheck_notrace: 4091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds RESTORE_REGS 410f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge addl $4, %esp # skip orig_eax/error_code 411fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 4120da5db313317e3195482d3e660a1074857374a89Rusty Russell1: INTERRUPT_RETURN 4131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.section .fixup,"ax" 4141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsiret_exc: 415a879cbbb34cbecfa9707fbb6e5a00c503ac1ecb9Linus Torvalds pushl $0 # no error code 416a879cbbb34cbecfa9707fbb6e5a00c503ac1ecb9Linus Torvalds pushl $do_iret_error 417a879cbbb34cbecfa9707fbb6e5a00c503ac1ecb9Linus Torvalds jmp error_code 4181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.previous 4191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.section __ex_table,"a" 4201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .align 4 4211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .long 1b,iret_exc 4221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.previous 4231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 424fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_RESTORE_STATE 4251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsldt_ss: 426eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge larl PT_OLDSS(%esp), %eax 4271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz restore_nocheck 4281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testl $0x00400000, %eax # returning to 32bit stack? 4291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz restore_nocheck # allright, normal return 430d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell 431d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell#ifdef CONFIG_PARAVIRT 432d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell /* 433d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * The kernel can't run on a non-flat stack if paravirt mode 434d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * is active. Rather than try to fixup the high bits of 435d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * ESP, bypass this code entirely. This may break DOSemu 436d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * and/or Wine support in a paravirt VM, although the option 437d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * is still available to implement the setting of the high 438d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell * 16-bits in the INTERRUPT_RETURN paravirt-op. 439d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell */ 44093b1eab3d29e7ea32ee583de3362da84db06ded8Jeremy Fitzhardinge cmpl $0, pv_info+PARAVIRT_enabled 441d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell jne restore_nocheck 442d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell#endif 443d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell 4441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* If returning to userspace with 16bit stack, 4451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * try to fix the higher word of ESP, as the CPU 4461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * won't restore it. 4471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This is an "official" bug of all the x86-compatible 4481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * CPUs, which we can try to work around to make 4491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * dosemu and wine happy. */ 450eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_OLDESP(%esp), %eax 451be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev movl %esp, %edx 452be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev call patch_espfix_desc 453be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev pushl $__ESPFIX_SS 454be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev CFI_ADJUST_CFA_OFFSET 4 455be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev pushl %eax 456be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev CFI_ADJUST_CFA_OFFSET 4 457139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_EAX) 45855f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 459be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev lss (%esp), %esp 460be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev CFI_ADJUST_CFA_OFFSET -8 461be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev jmp restore_nocheck 462fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 46347a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichENDPROC(system_call) 4641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # perform work that needs to be done immediately before resumption 4661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 467fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_PTREGS_FRAME # can't unwind into user space anyway 4681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldswork_pending: 4691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testb $_TIF_NEED_RESCHED, %cl 4701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz work_notifysig 4711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldswork_resched: 4721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call schedule 473c7e872e7da5514d014707a407ea562d197cc0136Peter Zijlstra LOCKDEP_SYS_EXIT 474139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 4751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # setting need_resched or sigpending 4761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # between sampling and the iret 47755f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 4781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl TI_flags(%ebp), %ecx 4791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds andl $_TIF_WORK_MASK, %ecx # is there any work to be done other 4801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # than syscall tracing? 4811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz restore_all 4821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testb $_TIF_NEED_RESCHED, %cl 4831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnz work_resched 4841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldswork_notifysig: # deal with pending signals and 4861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # notify-resume requests 48774b47a7844501445d41d704fe7c626f4b1819508Joe Korty#ifdef CONFIG_VM86 488eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge testl $VM_MASK, PT_EFLAGS(%esp) 4891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp, %eax 4901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne work_notifysig_v86 # returning to kernel-space or 4911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # vm86-space 4921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %edx, %edx 4931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_notify_resume 4944031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov jmp resume_userspace_sig 4951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 4971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldswork_notifysig_v86: 4981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ecx # save ti_flags for do_notify_resume 499fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 5001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call save_v86_state # %eax contains pt_regs pointer 5011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %ecx 502fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 5031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %eax, %esp 50474b47a7844501445d41d704fe7c626f4b1819508Joe Korty#else 50574b47a7844501445d41d704fe7c626f4b1819508Joe Korty movl %esp, %eax 50674b47a7844501445d41d704fe7c626f4b1819508Joe Korty#endif 5071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %edx, %edx 5081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_notify_resume 5094031ff388138b58e5cd472dccce38828bcb8c706Aleksey Gorelov jmp resume_userspace_sig 51047a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(work_pending) 5111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # perform syscall exit tracing 5131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 5141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_trace_entry: 515eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl $-ENOSYS,PT_EAX(%esp) 5161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp, %eax 5171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %edx,%edx 5181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_syscall_trace 519ed75e8d58010fdc06e2c3a81bfbebae92314c7e3Laurent Vivier cmpl $0, %eax 520640aa46e25922a00b805e6b0d0b5181ad9cf736aPaolo 'Blaisorblade' Giarrusso jne resume_userspace # ret != 0 -> running under PTRACE_SYSEMU, 521ed75e8d58010fdc06e2c3a81bfbebae92314c7e3Laurent Vivier # so must skip actual syscall 522eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_ORIG_EAX(%esp), %eax 5231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $(nr_syscalls), %eax 5241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jnae syscall_call 5251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp syscall_exit 52647a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(syscall_trace_entry) 5271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # perform syscall exit tracing 5291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 5301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_exit_work: 5311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl 5321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jz work_pending 53355f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_ON 534139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell ENABLE_INTERRUPTS(CLBR_ANY) # could let do_syscall_trace() call 5351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds # schedule() instead 5361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp, %eax 5371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl $1, %edx 5381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_syscall_trace 5391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp resume_userspace 54047a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(syscall_exit_work) 541fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 5421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 543fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME # can't unwind into user space anyway 5441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_fault: 5451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax # save orig_eax 546fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 5471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds SAVE_ALL 5481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds GET_THREAD_INFO(%ebp) 549eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl $-EFAULT,PT_EAX(%esp) 5501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp resume_userspace 55147a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(syscall_fault) 5521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_badsys: 554eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl $-ENOSYS,PT_EAX(%esp) 5551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp resume_userspace 55647a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(syscall_badsys) 557fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 5581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define FIXUP_ESPFIX_STACK \ 560be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev /* since we are on a wrong stack, we cant make it a C code :( */ \ 5617a61d35d4b4056e7711031202da7605e052f4137Jeremy Fitzhardinge PER_CPU(gdt_page, %ebx); \ 562be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah); \ 563be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev addl %esp, %eax; \ 564be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev pushl $__KERNEL_DS; \ 565be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev CFI_ADJUST_CFA_OFFSET 4; \ 5661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax; \ 567fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4; \ 568be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev lss (%esp), %esp; \ 569be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev CFI_ADJUST_CFA_OFFSET -8; 570be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev#define UNWIND_ESPFIX_STACK \ 5711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %ss, %eax; \ 572be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev /* see if on espfix stack */ \ 5731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpw $__ESPFIX_SS, %ax; \ 574be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev jne 27f; \ 575be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev movl $__KERNEL_DS, %eax; \ 576fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich movl %eax, %ds; \ 577fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich movl %eax, %es; \ 578be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev /* switch to normal stack */ \ 579fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich FIXUP_ESPFIX_STACK; \ 580be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev27:; 5811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 5831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Build the entry stubs and pointer table with 5841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * some assembler magic. 5851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 5861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.data 5871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(interrupt) 5881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.text 5891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(irq_entries_start) 591fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 59247a55cd795656d11bb18a7885583361f02a6baa8Jan Beulichvector=0 5931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.rept NR_IRQS 5941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 595fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich .if vector 596fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 597fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich .endif 59819eadf98c8167eac843580683317b99572e2abf0Rusty Russell1: pushl $~(vector) 599fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 6001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp common_interrupt 60147a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich .previous 6021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .long 1b 60347a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich .text 6041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvector=vector+1 6051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.endr 60647a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(irq_entries_start) 60747a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich 60847a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich.previous 60947a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(interrupt) 61047a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich.previous 6111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 61255f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar/* 61355f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar * the CPU automatically disables interrupts when executing an IRQ vector, 61455f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar * so IRQ-flags tracing has to follow that: 61555f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar */ 6161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 6171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldscommon_interrupt: 6181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds SAVE_ALL 61955f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF 6201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp,%eax 6211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_IRQ 6221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp ret_from_intr 62347a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichENDPROC(common_interrupt) 624fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 6251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define BUILD_INTERRUPT(name, nr) \ 6271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(name) \ 628fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME; \ 62919eadf98c8167eac843580683317b99572e2abf0Rusty Russell pushl $~(nr); \ 630fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4; \ 631fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich SAVE_ALL; \ 63255f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar TRACE_IRQS_OFF \ 6331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp,%eax; \ 634f76c392380a40008ee6ecaea4e5a51a3a10282c4Jeremy Fitzhardinge call smp_##name; \ 63555f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar jmp ret_from_intr; \ 63647a55cd795656d11bb18a7885583361f02a6baa8Jan Beulich CFI_ENDPROC; \ 63747a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichENDPROC(name) 6381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* The include is where all of the SMP etc. interrupts come from */ 6401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include "entry_arch.h" 6411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 642d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.PKPROBE_ENTRY(page_fault) 643d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P RING0_EC_FRAME 644d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P pushl $do_page_fault 645fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 6461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ALIGN 6471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldserror_code: 648464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge /* the function address is in %fs's slot on the stack */ 649f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge pushl %es 650f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET 4 651f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge /*CFI_REL_OFFSET es, 0*/ 6521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ds 653fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 654fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_REL_OFFSET ds, 0*/ 6551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax 656fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 657fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET eax, 0 6581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ebp 659fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 660fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET ebp, 0 6611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %edi 662fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 663fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET edi, 0 6641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %esi 665fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 666fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET esi, 0 6671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %edx 668fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 669fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET edx, 0 6701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ecx 671fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 672fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET ecx, 0 6731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ebx 674fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 675fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_REL_OFFSET ebx, 0 6761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cld 677464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge pushl %fs 678fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 679464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge /*CFI_REL_OFFSET fs, 0*/ 6807c3576d261ce046789a7db14f43303f8120910c7Jeremy Fitzhardinge movl $(__KERNEL_PERCPU), %ecx 681464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge movl %ecx, %fs 6821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds UNWIND_ESPFIX_STACK 6831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %ecx 684fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 685fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich /*CFI_REGISTER es, ecx*/ 686464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge movl PT_FS(%esp), %edi # get the function address 687eb5b7b9d86f46b45ba1f986302fdf7df84fb8297Jeremy Fitzhardinge movl PT_ORIG_EAX(%esp), %edx # get the error code 688f95d47caae5302a63d92be9a0292abc90e2a14e1Jeremy Fitzhardinge movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart 689464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge mov %ecx, PT_FS(%esp) 690464d1a78fbf8cf6c7fd970e7b3e2db50a320ce28Jeremy Fitzhardinge /*CFI_REL_OFFSET fs, ES*/ 6911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl $(__USER_DS), %ecx 6921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %ecx, %ds 6931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %ecx, %es 6941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp,%eax # pt_regs pointer 6951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call *%edi 6961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp ret_from_exception 697fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 698d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.PKPROBE_END(page_fault) 6991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(coprocessor_error) 701fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 7021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 703fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_coprocessor_error 705fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 707fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 70847a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(coprocessor_error) 7091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(simd_coprocessor_error) 711fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 7121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 713fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_simd_coprocessor_error 715fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 717fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 71847a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(simd_coprocessor_error) 7191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(device_not_available) 721fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 7221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $-1 # mark this as an int 723fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds SAVE_ALL 7250da5db313317e3195482d3e660a1074857374a89Rusty Russell GET_CR0_INTO_EAX 7261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds testl $0x4, %eax # EM (math emulation bit) 7271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne device_not_available_emulate 728139ec7c416248b9ea227d21839235344edfee1e0Rusty Russell preempt_stop(CLBR_ANY) 7291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call math_state_restore 7301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp ret_from_exception 7311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsdevice_not_available_emulate: 7321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 # temporary storage for ORIG_EIP 733fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call math_emulate 7351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds addl $4, %esp 736fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 7371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp ret_from_exception 738fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 73947a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(device_not_available) 7401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 7421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Debug traps and NMI can happen at the one SYSENTER instruction 7431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * that sets up the real kernel stack. Check here, since we can't 7441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * allow the wrong stack to be used. 7451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 746faca62273b602ab482fb7d3d940dbf41ef08b00eH. Peter Anvin * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have 7471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * already pushed 3 words if it hits on the sysenter instruction: 7481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * eflags, cs and eip. 7491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 7501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * We just load the right stack, and push the three (known) values 7511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * by hand onto the new stack - while updating the return eip past 7521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * the instruction that would have done it for sysenter. 7531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 7541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define FIX_STACK(offset, ok, label) \ 7551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpw $__KERNEL_CS,4(%esp); \ 7561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne ok; \ 7571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldslabel: \ 758faca62273b602ab482fb7d3d940dbf41ef08b00eH. Peter Anvin movl TSS_sysenter_sp0+offset(%esp),%esp; \ 759a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert CFI_DEF_CFA esp, 0; \ 760a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert CFI_UNDEFINED eip; \ 7611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushfl; \ 762a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert CFI_ADJUST_CFA_OFFSET 4; \ 7631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $__KERNEL_CS; \ 764a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert CFI_ADJUST_CFA_OFFSET 4; \ 765a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert pushl $sysenter_past_esp; \ 766a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert CFI_ADJUST_CFA_OFFSET 4; \ 767a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert CFI_REL_OFFSET eip, 0 7681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7693d97ae5b958855ac007b6f56a0f94ab8ade09e9ePrasanna S PanchamukhiKPROBE_ENTRY(debug) 770fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 7710aa97fb22624f18e5925d702ab0364d3838cfd91Roland McGrath cmpl $ia32_sysenter_target,(%esp) 7721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne debug_stack_correct 7731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn) 7741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsdebug_stack_correct: 7751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $-1 # mark this as an int 776fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds SAVE_ALL 7781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %edx,%edx # error code 0 7791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp,%eax # pt_regs pointer 7801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_debug 7811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp ret_from_exception 782fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 783d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.PKPROBE_END(debug) 784d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P 7851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 7861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * NMI is doubly nasty. It can happen _while_ we're handling 7871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * a debug fault, and the debug fault hasn't yet been able to 7881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * clear up the stack. So we first check whether we got an 7891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * NMI on the sysenter entry path, but after that we need to 7901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * check whether we got an NMI on the debug path where the debug 7911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * fault happened on the sysenter path. 7921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 79306039754d775d3e48e4a292e4f353321205eff53Fernando Luis V�zquez CaoKPROBE_ENTRY(nmi) 794fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 7951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax 796fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 7971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %ss, %eax 7981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpw $__ESPFIX_SS, %ax 7991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %eax 800fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 801be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev je nmi_espfix_stack 8020aa97fb22624f18e5925d702ab0364d3838cfd91Roland McGrath cmpl $ia32_sysenter_target,(%esp) 8031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds je nmi_stack_fixup 8041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax 805fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp,%eax 8071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Do not access memory above the end of our stack page, 8081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * it might not exist. 8091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 8101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds andl $(THREAD_SIZE-1),%eax 8111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $(THREAD_SIZE-20),%eax 8121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds popl %eax 813fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET -4 8141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jae nmi_stack_correct 8150aa97fb22624f18e5925d702ab0364d3838cfd91Roland McGrath cmpl $ia32_sysenter_target,12(%esp) 8161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds je nmi_debug_stack_check 8171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsnmi_stack_correct: 818a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert /* We have a RING0_INT_FRAME here */ 8191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax 820fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds SAVE_ALL 8221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %edx,%edx # zero error code 8231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp,%eax # pt_regs pointer 8241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_nmi 82555f327fa9e876758491a82af7491104f1cc3fc4dIngo Molnar jmp restore_nocheck_notrace 826fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 8271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsnmi_stack_fixup: 829a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert RING0_INT_FRAME 8301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds FIX_STACK(12,nmi_stack_correct, 1) 8311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp nmi_stack_correct 832a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert 8331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsnmi_debug_stack_check: 834a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert /* We have a RING0_INT_FRAME here */ 8351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpw $__KERNEL_CS,16(%esp) 8361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jne nmi_stack_correct 837e27182088e607880713d9c286a3d92d861c280e4Jan Beulich cmpl $debug,(%esp) 838e27182088e607880713d9c286a3d92d861c280e4Jan Beulich jb nmi_stack_correct 8391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cmpl $debug_esp_fix_insn,(%esp) 840e27182088e607880713d9c286a3d92d861c280e4Jan Beulich ja nmi_stack_correct 8411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds FIX_STACK(24,nmi_stack_correct, 1) 8421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp nmi_stack_correct 8431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 844be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeevnmi_espfix_stack: 845a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert /* We have a RING0_INT_FRAME here. 846a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert * 847a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert * create the pointer to lss back 848a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756Chuck Ebbert */ 8491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %ss 850fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %esp 852fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds addw $4, (%esp) 8541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* copy the iret frame of 12 bytes */ 8551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .rept 3 8561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl 16(%esp) 857fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .endr 8591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl %eax 860fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds SAVE_ALL 8621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds FIXUP_ESPFIX_STACK # %eax == %esp 8631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %edx,%edx # zero error code 8641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_nmi 8651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds RESTORE_REGS 866be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev lss 12+4(%esp), %esp # back to espfix stack 867be44d2aabce2d62f72d5751d1871b6212bf7a1c7Stas Sergeev CFI_ADJUST_CFA_OFFSET -24 8680da5db313317e3195482d3e660a1074857374a89Rusty Russell1: INTERRUPT_RETURN 869fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 8701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.section __ex_table,"a" 8711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .align 4 8721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds .long 1b,iret_exc 8731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds.previous 87406039754d775d3e48e4a292e4f353321205eff53Fernando Luis V�zquez CaoKPROBE_END(nmi) 8751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 876d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell#ifdef CONFIG_PARAVIRT 877d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty RussellENTRY(native_iret) 878d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell1: iret 879d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell.section __ex_table,"a" 880d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell .align 4 881d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell .long 1b,iret_exc 882d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell.previous 88347a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(native_iret) 884d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell 8856abcd98ffafbff81f0bfd7ee1d129e634af13245Glauber de Oliveira CostaENTRY(native_irq_enable_syscall_ret) 886d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell sti 887d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell sysexit 8886abcd98ffafbff81f0bfd7ee1d129e634af13245Glauber de Oliveira CostaEND(native_irq_enable_syscall_ret) 889d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell#endif 890d3561b7fa0fb0fc583bab0eeda32bec9e4c4056dRusty Russell 8913d97ae5b958855ac007b6f56a0f94ab8ade09e9ePrasanna S PanchamukhiKPROBE_ENTRY(int3) 892fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 8931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $-1 # mark this as an int 894fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 8951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds SAVE_ALL 8961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds xorl %edx,%edx # zero error code 8971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds movl %esp,%eax # pt_regs pointer 8981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds call do_int3 8991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp ret_from_exception 900fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 901d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.PKPROBE_END(int3) 9021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(overflow) 904fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 9051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 906fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_overflow 908fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 910fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 91147a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(overflow) 9121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(bounds) 914fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 9151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 916fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_bounds 918fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 920fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 92147a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(bounds) 9221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(invalid_op) 924fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 9251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 926fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_invalid_op 928fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 930fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 93147a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(invalid_op) 9321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(coprocessor_segment_overrun) 934fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 9351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 936fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_coprocessor_segment_overrun 938fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 940fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 94147a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(coprocessor_segment_overrun) 9421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(invalid_TSS) 944fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_EC_FRAME 9451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_invalid_TSS 946fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 948fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 94947a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(invalid_TSS) 9501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(segment_not_present) 952fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_EC_FRAME 9531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_segment_not_present 954fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 956fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 95747a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(segment_not_present) 9581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(stack_segment) 960fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_EC_FRAME 9611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_stack_segment 962fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 964fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 96547a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(stack_segment) 9661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9673d97ae5b958855ac007b6f56a0f94ab8ade09e9ePrasanna S PanchamukhiKPROBE_ENTRY(general_protection) 968fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_EC_FRAME 9691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_general_protection 970fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 972fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 973d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.PKPROBE_END(general_protection) 9741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(alignment_check) 976fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_EC_FRAME 9771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_alignment_check 978fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 980fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 98147a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(alignment_check) 9821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 983d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.PENTRY(divide_error) 984d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P RING0_INT_FRAME 985d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P pushl $0 # no error code 986d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P CFI_ADJUST_CFA_OFFSET 4 987d28c4393a7bf558538e9def269c1caeab6ec056fPrasanna S.P pushl $do_divide_error 988fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 990fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 99147a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(divide_error) 9921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef CONFIG_X86_MCE 9941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(machine_check) 995fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 9961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 997fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 9981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl machine_check_vector 999fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 10001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 1001fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 100247a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(machine_check) 10031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 10041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 10051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsENTRY(spurious_interrupt_bug) 1006fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich RING0_INT_FRAME 10071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $0 1008fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 10091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pushl $do_spurious_interrupt_bug 1010fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ADJUST_CFA_OFFSET 4 10111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds jmp error_code 1012fe7cacc1c25e286872b878c5d46880b620cd1e2dJan Beulich CFI_ENDPROC 101347a55cd795656d11bb18a7885583361f02a6baa8Jan BeulichEND(spurious_interrupt_bug) 10141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 101502ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi KleenENTRY(kernel_thread_helper) 101602ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen pushl $0 # fake return address for unwinder 101702ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen CFI_STARTPROC 101802ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen movl %edx,%eax 101902ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen push %edx 102002ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen CFI_ADJUST_CFA_OFFSET 4 102102ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen call *%ebx 102202ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen push %eax 102302ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen CFI_ADJUST_CFA_OFFSET 4 102402ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen call do_exit 102502ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen CFI_ENDPROC 102602ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi KleenENDPROC(kernel_thread_helper) 102702ba1a32dbd3d406530a17a2643a8f0f8cbf3accAndi Kleen 10285ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge#ifdef CONFIG_XEN 10295ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy FitzhardingeENTRY(xen_hypervisor_callback) 10305ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_STARTPROC 10315ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge pushl $0 10325ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET 4 10335ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge SAVE_ALL 10345ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge TRACE_IRQS_OFF 10359ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge 10369ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge /* Check to see if we got the event in the critical 10379ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge region in xen_iret_direct, after we've reenabled 10389ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge events and checked for pending events. This simulates 10399ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge iret instruction's behaviour where it delivers a 10409ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge pending interrupt when enabling interrupts. */ 10419ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge movl PT_EIP(%esp),%eax 10429ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge cmpl $xen_iret_start_crit,%eax 10439ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge jb 1f 10449ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge cmpl $xen_iret_end_crit,%eax 10459ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge jae 1f 10469ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge 10479ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge call xen_iret_crit_fixup 10489ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge 10499ec2b804e099e8a326369e6cccab10dee1d172eeJeremy Fitzhardinge1: mov %esp, %eax 10505ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge call xen_evtchn_do_upcall 10515ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp ret_from_intr 10525ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ENDPROC 10535ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy FitzhardingeENDPROC(xen_hypervisor_callback) 10545ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge 10555ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# Hypervisor uses this for application faults while it executes. 10565ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# We get here for two reasons: 10575ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# 1. Fault while reloading DS, ES, FS or GS 10585ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# 2. Fault while executing IRET 10595ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# Category 1 we fix up by reattempting the load, and zeroing the segment 10605ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# register if the load fails. 10615ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# Category 2 we fix up by jumping to do_iret_error. We cannot use the 10625ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# normal Linux return path in this case because if we use the IRET hypercall 10635ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# to pop the stack frame we end up in an infinite loop of failsafe callbacks. 10645ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge# We distinguish between categories by maintaining a status value in EAX. 10655ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy FitzhardingeENTRY(xen_failsafe_callback) 10665ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_STARTPROC 10675ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge pushl %eax 10685ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET 4 10695ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl $1,%eax 10705ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge1: mov 4(%esp),%ds 10715ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge2: mov 8(%esp),%es 10725ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge3: mov 12(%esp),%fs 10735ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge4: mov 16(%esp),%gs 10745ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge testl %eax,%eax 10755ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge popl %eax 10765ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET -4 10775ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge lea 16(%esp),%esp 10785ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET -16 10795ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jz 5f 10805ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge addl $16,%esp 10815ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp iret_exc # EAX != 0 => Category 2 (Bad IRET) 10825ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge5: pushl $0 # EAX == 0 => Category 1 (Bad segment) 10835ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ADJUST_CFA_OFFSET 4 10845ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge SAVE_ALL 10855ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp ret_from_exception 10865ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge CFI_ENDPROC 10875ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge 10885ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge.section .fixup,"ax" 10895ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge6: xorl %eax,%eax 10905ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl %eax,4(%esp) 10915ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp 1b 10925ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge7: xorl %eax,%eax 10935ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl %eax,8(%esp) 10945ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp 2b 10955ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge8: xorl %eax,%eax 10965ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl %eax,12(%esp) 10975ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp 3b 10985ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge9: xorl %eax,%eax 10995ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge movl %eax,16(%esp) 11005ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge jmp 4b 11015ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge.previous 11025ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge.section __ex_table,"a" 11035ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge .align 4 11045ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge .long 1b,6b 11055ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge .long 2b,7b 11065ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge .long 3b,8b 11075ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge .long 4b,9b 11085ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge.previous 11095ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy FitzhardingeENDPROC(xen_failsafe_callback) 11105ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge 11115ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge#endif /* CONFIG_XEN */ 11125ead97c84fa7d63a6a7a2f4e9f18f452bd109045Jeremy Fitzhardinge 1113bb152f53120d66c98c1f16518407df6a84f23714Arjan van de Ven.section .rodata,"a" 1114541054d935a1ec89916977cbf0d16ddb71b3ff5cThomas Gleixner#include "syscall_table_32.S" 11151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 11161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldssyscall_table_size=(.-sys_call_table) 1117