11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* arch/sparc64/mm/tlb.c 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (C) 2004 David S. Miller <davem@redhat.com> 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/kernel.h> 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/init.h> 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/percpu.h> 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mm.h> 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/swap.h> 11c9f2946fbec88d4baa3a6d47eb3a8e6b08b05cd9David S. Miller#include <linux/preempt.h> 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/pgtable.h> 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/pgalloc.h> 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/tlbflush.h> 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/cacheflush.h> 171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/mmu_context.h> 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/tlb.h> 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* Heavily inspired by the ppc64 code. */ 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2290f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstrastatic DEFINE_PER_CPU(struct tlb_batch, tlb_batch); 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid flush_tlb_pending(void) 251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2690f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra struct tlb_batch *tb = &get_cpu_var(tlb_batch); 271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2890f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra if (tb->tlb_nr) { 2990f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra flush_tsb_user(tb); 3074bf4312fff083ab25c3f357cc653ada7995e5f6David S. Miller 3190f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra if (CTX_VALID(tb->mm->context)) { 321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef CONFIG_SMP 3390f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra smp_flush_tlb_pending(tb->mm, tb->tlb_nr, 3490f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra &tb->vaddrs[0]); 351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#else 3690f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra __flush_tlb_pending(CTX_HWBITS(tb->mm->context), 3790f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra tb->tlb_nr, &tb->vaddrs[0]); 381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 4090f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra tb->tlb_nr = 0; 411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 42c9f2946fbec88d4baa3a6d47eb3a8e6b08b05cd9David S. Miller 4390f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra put_cpu_var(tlb_batch); 441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4690f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstravoid tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, 4790f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra pte_t *ptep, pte_t orig, int fullmm) 481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 4990f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra struct tlb_batch *tb = &get_cpu_var(tlb_batch); 501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long nr; 511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds vaddr &= PAGE_MASK; 531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (pte_exec(orig)) 541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds vaddr |= 0x1UL; 551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 567a591cfe4efef8a232e4938d44ae6693b319f6d7David S. Miller if (tlb_type != hypervisor && 577a591cfe4efef8a232e4938d44ae6693b319f6d7David S. Miller pte_dirty(orig)) { 581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long paddr, pfn = pte_pfn(orig); 591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct address_space *mapping; 601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page *page; 611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!pfn_valid(pfn)) 631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto no_cache_flush; 641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds page = pfn_to_page(pfn); 661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (PageReserved(page)) 671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto no_cache_flush; 681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* A real file page? */ 701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mapping = page_mapping(page); 711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!mapping) 721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto no_cache_flush; 731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds paddr = (unsigned long) page_address(page); 751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if ((paddr ^ vaddr) & (1 << 13)) 761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flush_dcache_page_all(mm, page); 771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsno_cache_flush: 801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8190f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra if (fullmm) { 8290f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra put_cpu_var(tlb_batch); 831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return; 8490f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra } 851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8690f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra nr = tb->tlb_nr; 871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8890f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra if (unlikely(nr != 0 && mm != tb->mm)) { 891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flush_tlb_pending(); 901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds nr = 0; 911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (nr == 0) 9490f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra tb->mm = mm; 951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9690f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra tb->vaddrs[nr] = vaddr; 9790f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra tb->tlb_nr = ++nr; 981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (nr >= TLB_BATCH_NR) 991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flush_tlb_pending(); 10090f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra 10190f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra put_cpu_var(tlb_batch); 1021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 103