11965aae3c98397aad957412413c07e97b1bd4e64H. Peter Anvin#ifndef _ASM_X86_TLBFLUSH_H 21965aae3c98397aad957412413c07e97b1bd4e64H. Peter Anvin#define _ASM_X86_TLBFLUSH_H 3d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 4d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#include <linux/mm.h> 5d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#include <linux/sched.h> 6d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 7d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#include <asm/processor.h> 8f05e798ad4c09255f590f5b2c00a7ca6c172f983David Howells#include <asm/special_insns.h> 9d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 10d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#ifdef CONFIG_PARAVIRT 11d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#include <asm/paravirt.h> 12d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#else 13d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#define __flush_tlb() __native_flush_tlb() 14d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#define __flush_tlb_global() __native_flush_tlb_global() 15d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#define __flush_tlb_single(addr) __native_flush_tlb_single(addr) 16d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#endif 17d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 18d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixnerstatic inline void __native_flush_tlb(void) 19d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner{ 20d7285c6b5c54397fdf112c2fb98ee43193173aa9Chris Wright native_write_cr3(native_read_cr3()); 21d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner} 22d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 23d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixnerstatic inline void __native_flush_tlb_global(void) 24d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner{ 25b1979a5fda7869a790f4fd83fb06c78498d26ba1Ingo Molnar unsigned long flags; 26b1979a5fda7869a790f4fd83fb06c78498d26ba1Ingo Molnar unsigned long cr4; 27d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 28b1979a5fda7869a790f4fd83fb06c78498d26ba1Ingo Molnar /* 29b1979a5fda7869a790f4fd83fb06c78498d26ba1Ingo Molnar * Read-modify-write to CR4 - protect it from preemption and 30b1979a5fda7869a790f4fd83fb06c78498d26ba1Ingo Molnar * from interrupts. (Use the raw variant because this code can 31b1979a5fda7869a790f4fd83fb06c78498d26ba1Ingo Molnar * be called from deep inside debugging code.) 32b1979a5fda7869a790f4fd83fb06c78498d26ba1Ingo Molnar */ 33b1979a5fda7869a790f4fd83fb06c78498d26ba1Ingo Molnar raw_local_irq_save(flags); 34b1979a5fda7869a790f4fd83fb06c78498d26ba1Ingo Molnar 35d7285c6b5c54397fdf112c2fb98ee43193173aa9Chris Wright cr4 = native_read_cr4(); 36d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner /* clear PGE */ 37d7285c6b5c54397fdf112c2fb98ee43193173aa9Chris Wright native_write_cr4(cr4 & ~X86_CR4_PGE); 38d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner /* write old PGE again and flush TLBs */ 39d7285c6b5c54397fdf112c2fb98ee43193173aa9Chris Wright native_write_cr4(cr4); 40b1979a5fda7869a790f4fd83fb06c78498d26ba1Ingo Molnar 41b1979a5fda7869a790f4fd83fb06c78498d26ba1Ingo Molnar raw_local_irq_restore(flags); 42d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner} 43d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 44d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixnerstatic inline void __native_flush_tlb_single(unsigned long addr) 45d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner{ 4694cf8de0a0aff39c7b7785af4fc938ecacb79b7cJoe Perches asm volatile("invlpg (%0)" ::"r" (addr) : "memory"); 47d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner} 48d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 49d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixnerstatic inline void __flush_tlb_all(void) 50d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner{ 51d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner if (cpu_has_pge) 52d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner __flush_tlb_global(); 53d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner else 54d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner __flush_tlb(); 55d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner} 56d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 57d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixnerstatic inline void __flush_tlb_one(unsigned long addr) 58d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner{ 59d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner if (cpu_has_invlpg) 60d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner __flush_tlb_single(addr); 61d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner else 62d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner __flush_tlb(); 63d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner} 64d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 6596a388de5dc53a8b234b3fd41f3ae2cedc9ffd42Thomas Gleixner#ifdef CONFIG_X86_32 66d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner# define TLB_FLUSH_ALL 0xffffffff 6796a388de5dc53a8b234b3fd41f3ae2cedc9ffd42Thomas Gleixner#else 68d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner# define TLB_FLUSH_ALL -1ULL 69d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#endif 70d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 71d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner/* 72d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner * TLB flushing: 73d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner * 74d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner * - flush_tlb() flushes the current mm struct TLBs 75d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner * - flush_tlb_all() flushes all processes TLBs 76d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner * - flush_tlb_mm(mm) flushes the specified mm context TLB's 77d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner * - flush_tlb_page(vma, vmaddr) flushes one page 78d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner * - flush_tlb_range(vma, start, end) flushes a range of pages 79d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages 80d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner * - flush_tlb_others(cpumask, mm, va) flushes TLBs on other cpus 81d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner * 82d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner * ..but the i386 has somewhat limited tlb flushing capabilities, 83d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner * and page-granular flushes are available only on i486 and up. 84d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner * 85d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner * x86-64 can only flush individual pages or full VMs. For a range flush 86d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner * we always do the full VM. Might be worth trying if for a small 87d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner * range a few INVLPGs in a row are a win. 88d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner */ 89d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 90d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#ifndef CONFIG_SMP 91d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 92d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#define flush_tlb() __flush_tlb() 93d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#define flush_tlb_all() __flush_tlb_all() 94d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#define local_flush_tlb() __flush_tlb() 95d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 96d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixnerstatic inline void flush_tlb_mm(struct mm_struct *mm) 97d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner{ 98d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner if (mm == current->active_mm) 99d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner __flush_tlb(); 100d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner} 101d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 102d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixnerstatic inline void flush_tlb_page(struct vm_area_struct *vma, 103d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner unsigned long addr) 104d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner{ 105d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner if (vma->vm_mm == current->active_mm) 106d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner __flush_tlb_one(addr); 107d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner} 108d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 109d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixnerstatic inline void flush_tlb_range(struct vm_area_struct *vma, 110d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner unsigned long start, unsigned long end) 111d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner{ 112d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner if (vma->vm_mm == current->active_mm) 113d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner __flush_tlb(); 114d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner} 115d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 1164595f9620cda8a1e973588e743cf5f8436dd20c6Rusty Russellstatic inline void native_flush_tlb_others(const struct cpumask *cpumask, 117d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner struct mm_struct *mm, 118d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner unsigned long va) 119d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner{ 120d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner} 121d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 122913da64b54b2b3bb212a59aba2e6f2b8294ca1faAlex Nixonstatic inline void reset_lazy_tlbstate(void) 123913da64b54b2b3bb212a59aba2e6f2b8294ca1faAlex Nixon{ 124913da64b54b2b3bb212a59aba2e6f2b8294ca1faAlex Nixon} 125913da64b54b2b3bb212a59aba2e6f2b8294ca1faAlex Nixon 126d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#else /* SMP */ 127d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 128d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#include <asm/smp.h> 129d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 130d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#define local_flush_tlb() __flush_tlb() 131d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 132d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixnerextern void flush_tlb_all(void); 133d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixnerextern void flush_tlb_current_task(void); 134d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixnerextern void flush_tlb_mm(struct mm_struct *); 135d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixnerextern void flush_tlb_page(struct vm_area_struct *, unsigned long); 136d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 137d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#define flush_tlb() flush_tlb_current_task() 138d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 139d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixnerstatic inline void flush_tlb_range(struct vm_area_struct *vma, 140d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner unsigned long start, unsigned long end) 141d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner{ 142d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner flush_tlb_mm(vma->vm_mm); 143d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner} 144d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 1454595f9620cda8a1e973588e743cf5f8436dd20c6Rusty Russellvoid native_flush_tlb_others(const struct cpumask *cpumask, 1464595f9620cda8a1e973588e743cf5f8436dd20c6Rusty Russell struct mm_struct *mm, unsigned long va); 147d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 148d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#define TLBSTATE_OK 1 149d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#define TLBSTATE_LAZY 2 150d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 15194cf8de0a0aff39c7b7785af4fc938ecacb79b7cJoe Perchesstruct tlb_state { 152d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner struct mm_struct *active_mm; 153d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner int state; 154d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner}; 1559b8de7479d0dbab1ed98b5b015d44232c9d3d08eDavid HowellsDECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate); 156913da64b54b2b3bb212a59aba2e6f2b8294ca1faAlex Nixon 157913da64b54b2b3bb212a59aba2e6f2b8294ca1faAlex Nixonstatic inline void reset_lazy_tlbstate(void) 158913da64b54b2b3bb212a59aba2e6f2b8294ca1faAlex Nixon{ 1599eb912d1aa6b8106e06a73ea6702ec3dab0d6a1aBrian Gerst percpu_write(cpu_tlbstate.state, 0); 1609eb912d1aa6b8106e06a73ea6702ec3dab0d6a1aBrian Gerst percpu_write(cpu_tlbstate.active_mm, &init_mm); 161913da64b54b2b3bb212a59aba2e6f2b8294ca1faAlex Nixon} 162d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 163d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#endif /* SMP */ 164d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 165d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner#ifndef CONFIG_PARAVIRT 1664595f9620cda8a1e973588e743cf5f8436dd20c6Rusty Russell#define flush_tlb_others(mask, mm, va) native_flush_tlb_others(mask, mm, va) 16796a388de5dc53a8b234b3fd41f3ae2cedc9ffd42Thomas Gleixner#endif 168d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 169d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixnerstatic inline void flush_tlb_kernel_range(unsigned long start, 170d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner unsigned long end) 171d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner{ 172d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner flush_tlb_all(); 173d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner} 174d291cf83639a0e0b67ff783b6ed29c0a747d4901Thomas Gleixner 1751965aae3c98397aad957412413c07e97b1bd4e64H. Peter Anvin#endif /* _ASM_X86_TLBFLUSH_H */ 176