11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifndef __UM_TLB_H 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define __UM_TLB_H 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 41f4deba80a0d9fa92832684e683335b742a530e8WANG Cong#include <linux/pagemap.h> 50b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike#include <linux/swap.h> 60b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike#include <asm/percpu.h> 70b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike#include <asm/pgalloc.h> 80b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike#include <asm/tlbflush.h> 90b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike 100b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike#define tlb_start_vma(tlb, vma) do { } while (0) 110b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike#define tlb_end_vma(tlb, vma) do { } while (0) 120b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm) 130b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike 140b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike/* struct mmu_gather is an opaque type used by the mm code for passing around 150b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike * any data needed by arch specific code for tlb_remove_page. 160b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike */ 170b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dikestruct mmu_gather { 180b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike struct mm_struct *mm; 190b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike unsigned int need_flush; /* Really unmapped some ptes? */ 200b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike unsigned long start; 210b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike unsigned long end; 220b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike unsigned int fullmm; /* non-zero means full mm flush */ 230b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike}; 240b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike 250b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dikestatic inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, 260b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike unsigned long address) 270b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike{ 280b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike if (tlb->start > address) 290b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike tlb->start = address; 300b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike if (tlb->end < address + PAGE_SIZE) 310b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike tlb->end = address + PAGE_SIZE; 320b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike} 330b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike 340b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dikestatic inline void init_tlb_gather(struct mmu_gather *tlb) 350b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike{ 360b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike tlb->need_flush = 0; 370b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike 380b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike tlb->start = TASK_SIZE; 390b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike tlb->end = 0; 400b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike 410b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike if (tlb->fullmm) { 420b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike tlb->start = 0; 430b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike tlb->end = TASK_SIZE; 440b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike } 450b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike} 460b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike 47ff075d605511784c79cbf0ae73d90e07238267b3Peter Zijlstrastatic inline void 48ff075d605511784c79cbf0ae73d90e07238267b3Peter Zijlstratlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned int full_mm_flush) 490b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike{ 500b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike tlb->mm = mm; 510b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike tlb->fullmm = full_mm_flush; 520b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike 530b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike init_tlb_gather(tlb); 540b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike} 550b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike 560b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dikeextern void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, 570b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike unsigned long end); 580b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike 590b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dikestatic inline void 60ff075d605511784c79cbf0ae73d90e07238267b3Peter Zijlstratlb_flush_mmu(struct mmu_gather *tlb) 610b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike{ 620b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike if (!tlb->need_flush) 630b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike return; 640b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike 650b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end); 660b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike init_tlb_gather(tlb); 670b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike} 680b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike 690b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike/* tlb_finish_mmu 700b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike * Called at the end of the shootdown operation to free up any resources 710b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike * that were required. 720b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike */ 730b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dikestatic inline void 740b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Diketlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) 750b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike{ 76ff075d605511784c79cbf0ae73d90e07238267b3Peter Zijlstra tlb_flush_mmu(tlb); 770b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike 780b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike /* keep the page table cache within bounds */ 790b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike check_pgt_cache(); 800b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike} 810b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike 820b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike/* tlb_remove_page 830b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike * Must perform the equivalent to __free_pte(pte_get_and_clear(ptep)), 840b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike * while handling the additional races in SMP caused by other CPUs 850b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike * caching valid mappings in their TLBs. 860b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike */ 87ff075d605511784c79cbf0ae73d90e07238267b3Peter Zijlstrastatic inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page) 880b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike{ 890b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike tlb->need_flush = 1; 900b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike free_page_and_swap_cache(page); 91ff075d605511784c79cbf0ae73d90e07238267b3Peter Zijlstra return 1; /* avoid calling tlb_flush_mmu */ 92ff075d605511784c79cbf0ae73d90e07238267b3Peter Zijlstra} 93ff075d605511784c79cbf0ae73d90e07238267b3Peter Zijlstra 94ff075d605511784c79cbf0ae73d90e07238267b3Peter Zijlstrastatic inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) 95ff075d605511784c79cbf0ae73d90e07238267b3Peter Zijlstra{ 96ff075d605511784c79cbf0ae73d90e07238267b3Peter Zijlstra __tlb_remove_page(tlb, page); 970b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike} 980b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike 990b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike/** 1000b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike * tlb_remove_tlb_entry - remember a pte unmapping for later tlb invalidation. 1010b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike * 1020b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike * Record the fact that pte's were really umapped in ->need_flush, so we can 1030b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike * later optimise away the tlb invalidate. This helps when userspace is 1040b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike * unmapping already-unmapped pages, which happens quite a lot. 1050b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike */ 1060b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike#define tlb_remove_tlb_entry(tlb, ptep, address) \ 1070b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike do { \ 1080b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike tlb->need_flush = 1; \ 1090b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike __tlb_remove_tlb_entry(tlb, ptep, address); \ 1100b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike } while (0) 1110b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike 1129e1b32caa525cb236e80e9c671e179bcecccc657Benjamin Herrenschmidt#define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr) 1130b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike 1149e1b32caa525cb236e80e9c671e179bcecccc657Benjamin Herrenschmidt#define pud_free_tlb(tlb, pudp, addr) __pud_free_tlb(tlb, pudp, addr) 1150b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike 1169e1b32caa525cb236e80e9c671e179bcecccc657Benjamin Herrenschmidt#define pmd_free_tlb(tlb, pmdp, addr) __pmd_free_tlb(tlb, pmdp, addr) 1170b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike 1180b4e273fb83bce5dd8e166a4defb16ebdd215abfJeff Dike#define tlb_migrate_finish(mm) do {} while (0) 1191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 121