1c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru/* asm-generic/tlb.h 2c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * 3c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * Generic TLB shootdown code 4c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * 5c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * Copyright 2001 Red Hat, Inc. 6c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * Based on code from mm/memory.c Copyright Linus Torvalds and others. 7c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * 8c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * This program is free software; you can redistribute it and/or 9c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * modify it under the terms of the GNU General Public License 10c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * as published by the Free Software Foundation; either version 11c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * 2 of the License, or (at your option) any later version. 12c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru */ 13c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#ifndef _ASM_GENERIC__TLB_H 14c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define _ASM_GENERIC__TLB_H 15c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 16c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#include <linux/swap.h> 17c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#include <asm/pgalloc.h> 18c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#include <asm/tlbflush.h> 19c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 20c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru/* 21c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * For UP we don't need to worry about TLB flush 22c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * and page free order so much.. 23c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru */ 24c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#ifdef CONFIG_SMP 25c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru #ifdef ARCH_FREE_PTR_NR 26c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru #define FREE_PTR_NR ARCH_FREE_PTR_NR 27c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru #else 28c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru #define FREE_PTE_NR 506 29c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru #endif 30c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru #define tlb_fast_mode(tlb) ((tlb)->nr == ~0U) 31c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#else 32c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru #define FREE_PTE_NR 1 33c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru #define tlb_fast_mode(tlb) 1 34c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#endif 35c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 36c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru/* struct mmu_gather is an opaque type used by the mm code for passing around 37c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * any data needed by arch specific code for tlb_remove_page. 38c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru */ 39c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Querustruct mmu_gather { 40c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru struct mm_struct *mm; 41c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned int nr; /* set to ~0U means fast mode */ 42c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned int need_flush;/* Really unmapped some ptes? */ 43c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned int fullmm; /* non-zero means full mm flush */ 44c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru struct page * pages[FREE_PTE_NR]; 45c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru}; 46c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 47c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru/* Users of the generic TLB shootdown code must declare this storage space. */ 48c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste QueruDECLARE_PER_CPU(struct mmu_gather, mmu_gathers); 49c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 50c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru/* tlb_gather_mmu 51c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * Return a pointer to an initialized struct mmu_gather. 52c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru */ 53c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Querustatic inline struct mmu_gather * 54c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Querutlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush) 55c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru{ 56c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru struct mmu_gather *tlb = &get_cpu_var(mmu_gathers); 57c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 58c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru tlb->mm = mm; 59c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 60c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru /* Use fast mode if only one CPU is online */ 61c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru tlb->nr = num_online_cpus() > 1 ? 0U : ~0U; 62c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 63c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru tlb->fullmm = full_mm_flush; 64c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 65c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru return tlb; 66c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru} 67c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 68c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Querustatic inline void 69c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Querutlb_flush_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) 70c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru{ 71c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru if (!tlb->need_flush) 72c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru return; 73c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru tlb->need_flush = 0; 74c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru tlb_flush(tlb); 75c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru if (!tlb_fast_mode(tlb)) { 76c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru free_pages_and_swap_cache(tlb->pages, tlb->nr); 77c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru tlb->nr = 0; 78c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru } 79c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru} 80c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 81c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru/* tlb_finish_mmu 82c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * Called at the end of the shootdown operation to free up any resources 83c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * that were required. 84c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru */ 85c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Querustatic inline void 86c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Querutlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) 87c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru{ 88c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru tlb_flush_mmu(tlb, start, end); 89c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 90c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru /* keep the page table cache within bounds */ 91c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru check_pgt_cache(); 92c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 93c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru put_cpu_var(mmu_gathers); 94c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru} 95c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 96c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru/* tlb_remove_page 97c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * Must perform the equivalent to __free_pte(pte_get_and_clear(ptep)), while 98c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * handling the additional races in SMP caused by other CPUs caching valid 99c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * mappings in their TLBs. 100c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru */ 101c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Querustatic inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) 102c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru{ 103c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru tlb->need_flush = 1; 104c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru if (tlb_fast_mode(tlb)) { 105c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru free_page_and_swap_cache(page); 106c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru return; 107c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru } 108c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru tlb->pages[tlb->nr++] = page; 109c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru if (tlb->nr >= FREE_PTE_NR) 110c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru tlb_flush_mmu(tlb, 0, 0); 111c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru} 112c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 113c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru/** 114c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * tlb_remove_tlb_entry - remember a pte unmapping for later tlb invalidation. 115c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * 116c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * Record the fact that pte's were really umapped in ->need_flush, so we can 117c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * later optimise away the tlb invalidate. This helps when userspace is 118c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * unmapping already-unmapped pages, which happens quite a lot. 119c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru */ 120c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define tlb_remove_tlb_entry(tlb, ptep, address) \ 121c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru do { \ 122c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru tlb->need_flush = 1; \ 123c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru __tlb_remove_tlb_entry(tlb, ptep, address); \ 124c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru } while (0) 125c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 126c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define pte_free_tlb(tlb, ptep) \ 127c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru do { \ 128c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru tlb->need_flush = 1; \ 129c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru __pte_free_tlb(tlb, ptep); \ 130c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru } while (0) 131c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 132c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#ifndef __ARCH_HAS_4LEVEL_HACK 133c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define pud_free_tlb(tlb, pudp) \ 134c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru do { \ 135c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru tlb->need_flush = 1; \ 136c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru __pud_free_tlb(tlb, pudp); \ 137c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru } while (0) 138c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#endif 139c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 140c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define pmd_free_tlb(tlb, pmdp) \ 141c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru do { \ 142c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru tlb->need_flush = 1; \ 143c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru __pmd_free_tlb(tlb, pmdp); \ 144c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru } while (0) 145c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 146c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define tlb_migrate_finish(mm) do {} while (0) 147c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 148c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#endif /* _ASM_GENERIC__TLB_H */ 149