cache.c revision 6b2f3d1f769be5779b479c37800229d9a4809fc3
11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This file is subject to the terms and conditions of the GNU General Public 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * License. See the file "COPYING" in the main directory of this archive 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * for more details. 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 6641e97f318870921d048154af6807e46e43c307aRalf Baechle * Copyright (C) 1994 - 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org) 77575a49f209190ca640e0da792565a1bcb641f3eRalf Baechle * Copyright (C) 2007 MIPS Technologies, Inc. 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 924e9d0b96dac5503c0b6f034d553030c604228a7Ralf Baechle#include <linux/fs.h> 1024e9d0b96dac5503c0b6f034d553030c604228a7Ralf Baechle#include <linux/fcntl.h> 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/init.h> 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/kernel.h> 13641e97f318870921d048154af6807e46e43c307aRalf Baechle#include <linux/linkage.h> 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/module.h> 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/sched.h> 16dbda6ac0897603f6c6dfadbbc37f9882177ec7acRalf Baechle#include <linux/syscalls.h> 171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mm.h> 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/cacheflush.h> 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/processor.h> 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/cpu.h> 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/cpu-features.h> 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* Cache operations. */ 251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid (*flush_cache_all)(void); 261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid (*__flush_cache_all)(void); 271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid (*flush_cache_mm)(struct mm_struct *mm); 281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid (*flush_cache_range)(struct vm_area_struct *vma, unsigned long start, 291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long end); 3053de0d471fe8ddbbeca938cffedb4cc94e04da10Ralf Baechlevoid (*flush_cache_page)(struct vm_area_struct *vma, unsigned long page, 3153de0d471fe8ddbbeca938cffedb4cc94e04da10Ralf Baechle unsigned long pfn); 32d4264f183967db9c2dae4275abb98eb1f79facb2Atsushi Nemotovoid (*flush_icache_range)(unsigned long start, unsigned long end); 33e0cee3eea7875800451739ae38f99edcf11c133dThomas Bogendoerfervoid (*local_flush_icache_range)(unsigned long start, unsigned long end); 341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 359c5a3d729cf430609d091ff610a7db363aafcd47Ralf Baechlevoid (*__flush_cache_vmap)(void); 369c5a3d729cf430609d091ff610a7db363aafcd47Ralf Baechlevoid (*__flush_cache_vunmap)(void); 379c5a3d729cf430609d091ff610a7db363aafcd47Ralf Baechle 381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* MIPS specific cache operations */ 391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid (*flush_cache_sigtramp)(unsigned long addr); 407e3bfc7cfc402458b0386086ab650ce811720927Ralf Baechlevoid (*local_flush_data_cache_page)(void * addr); 411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid (*flush_data_cache_page)(unsigned long addr); 421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid (*flush_icache_all)(void); 431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 449202f32558601c2c99ddc438eb3218131d00d413Ralf BaechleEXPORT_SYMBOL_GPL(local_flush_data_cache_page); 459ff77c469ed16221c6a4e882e48e4f0dcf451bdaRalf BaechleEXPORT_SYMBOL(flush_data_cache_page); 469ff77c469ed16221c6a4e882e48e4f0dcf451bdaRalf Baechle 471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef CONFIG_DMA_NONCOHERENT 481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* DMA cache operations. */ 501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid (*_dma_cache_wback_inv)(unsigned long start, unsigned long size); 511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid (*_dma_cache_wback)(unsigned long start, unsigned long size); 521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid (*_dma_cache_inv)(unsigned long start, unsigned long size); 531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(_dma_cache_wback_inv); 551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif /* CONFIG_DMA_NONCOHERENT */ 571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * We could optimize the case where the cache argument is not BCACHE but 601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * that seems very atypical use ... 611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 62dbda6ac0897603f6c6dfadbbc37f9882177ec7acRalf BaechleSYSCALL_DEFINE3(cacheflush, unsigned long, addr, unsigned long, bytes, 63dbda6ac0897603f6c6dfadbbc37f9882177ec7acRalf Baechle unsigned int, cache) 641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 65750ccf687ff9adbf2a16066a3a2757d0f761384cAtsushi Nemoto if (bytes == 0) 66750ccf687ff9adbf2a16066a3a2757d0f761384cAtsushi Nemoto return 0; 67fe00f943e0ef98b4057abcc2940d631a975b43cdRalf Baechle if (!access_ok(VERIFY_WRITE, (void __user *) addr, bytes)) 681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EFAULT; 691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flush_icache_range(addr, addr + bytes); 711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid __flush_dcache_page(struct page *page) 761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct address_space *mapping = page_mapping(page); 781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long addr; 791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 80585fa72493edd7d5acb308806e7bb609412c6228Ralf Baechle if (PageHighMem(page)) 81585fa72493edd7d5acb308806e7bb609412c6228Ralf Baechle return; 821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (mapping && !mapping_mapped(mapping)) { 831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds SetPageDcacheDirty(page); 841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return; 851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * We could delay the flush for the !page_mapping case too. But that 891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * case is for exec env/arg pages and those are %99 certainly going to 901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * get faulted into the tlb (and thus flushed) anyways. 911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds addr = (unsigned long) page_address(page); 931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flush_data_cache_page(addr); 941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(__flush_dcache_page); 971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 987575a49f209190ca640e0da792565a1bcb641f3eRalf Baechlevoid __flush_anon_page(struct page *page, unsigned long vmaddr) 997575a49f209190ca640e0da792565a1bcb641f3eRalf Baechle{ 1009a74b3eb22f2d67a5681301f52aca5b7703382c8Ralf Baechle unsigned long addr = (unsigned long) page_address(page); 1017575a49f209190ca640e0da792565a1bcb641f3eRalf Baechle 1029a74b3eb22f2d67a5681301f52aca5b7703382c8Ralf Baechle if (pages_do_alias(addr, vmaddr)) { 1039a74b3eb22f2d67a5681301f52aca5b7703382c8Ralf Baechle if (page_mapped(page) && !Page_dcache_dirty(page)) { 1049a74b3eb22f2d67a5681301f52aca5b7703382c8Ralf Baechle void *kaddr; 1059a74b3eb22f2d67a5681301f52aca5b7703382c8Ralf Baechle 1069a74b3eb22f2d67a5681301f52aca5b7703382c8Ralf Baechle kaddr = kmap_coherent(page, vmaddr); 1079a74b3eb22f2d67a5681301f52aca5b7703382c8Ralf Baechle flush_data_cache_page((unsigned long)kaddr); 1089a74b3eb22f2d67a5681301f52aca5b7703382c8Ralf Baechle kunmap_coherent(); 1099a74b3eb22f2d67a5681301f52aca5b7703382c8Ralf Baechle } else 1109a74b3eb22f2d67a5681301f52aca5b7703382c8Ralf Baechle flush_data_cache_page(addr); 1117575a49f209190ca640e0da792565a1bcb641f3eRalf Baechle } 1127575a49f209190ca640e0da792565a1bcb641f3eRalf Baechle} 1137575a49f209190ca640e0da792565a1bcb641f3eRalf Baechle 1147575a49f209190ca640e0da792565a1bcb641f3eRalf BaechleEXPORT_SYMBOL(__flush_anon_page); 1157575a49f209190ca640e0da792565a1bcb641f3eRalf Baechle 1161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid __update_cache(struct vm_area_struct *vma, unsigned long address, 1171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pte_t pte) 1181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page *page; 1201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long pfn, addr; 121585fa72493edd7d5acb308806e7bb609412c6228Ralf Baechle int exec = (vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc; 1221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pfn = pte_pfn(pte); 124585fa72493edd7d5acb308806e7bb609412c6228Ralf Baechle if (unlikely(!pfn_valid(pfn))) 125585fa72493edd7d5acb308806e7bb609412c6228Ralf Baechle return; 126585fa72493edd7d5acb308806e7bb609412c6228Ralf Baechle page = pfn_to_page(pfn); 127585fa72493edd7d5acb308806e7bb609412c6228Ralf Baechle if (page_mapping(page) && Page_dcache_dirty(page)) { 128585fa72493edd7d5acb308806e7bb609412c6228Ralf Baechle addr = (unsigned long) page_address(page); 129585fa72493edd7d5acb308806e7bb609412c6228Ralf Baechle if (exec || pages_do_alias(addr, address & PAGE_MASK)) 1301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flush_data_cache_page(addr); 1311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ClearPageDcacheDirty(page); 1321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 135351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris Dearmanunsigned long _page_cachable_default; 136351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris DearmanEXPORT_SYMBOL_GPL(_page_cachable_default); 137351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris Dearman 138351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris Dearmanstatic inline void setup_protection_map(void) 139351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris Dearman{ 140351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris Dearman protection_map[0] = PAGE_NONE; 141351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris Dearman protection_map[1] = PAGE_READONLY; 142351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris Dearman protection_map[2] = PAGE_COPY; 143351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris Dearman protection_map[3] = PAGE_COPY; 144351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris Dearman protection_map[4] = PAGE_READONLY; 145351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris Dearman protection_map[5] = PAGE_READONLY; 146351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris Dearman protection_map[6] = PAGE_COPY; 147351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris Dearman protection_map[7] = PAGE_COPY; 148351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris Dearman protection_map[8] = PAGE_NONE; 149351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris Dearman protection_map[9] = PAGE_READONLY; 150351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris Dearman protection_map[10] = PAGE_SHARED; 151351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris Dearman protection_map[11] = PAGE_SHARED; 152351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris Dearman protection_map[12] = PAGE_READONLY; 153351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris Dearman protection_map[13] = PAGE_READONLY; 154351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris Dearman protection_map[14] = PAGE_SHARED; 155351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris Dearman protection_map[15] = PAGE_SHARED; 156351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris Dearman} 1571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 158234fcd1484a66158b561b36b421547f0ab85fee9Ralf Baechlevoid __devinit cpu_cache_init(void) 1591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 16002cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle if (cpu_has_3k_cache) { 16102cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle extern void __weak r3k_cache_init(void); 16202cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle 16302cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle r3k_cache_init(); 16402cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle } 16502cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle if (cpu_has_6k_cache) { 16602cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle extern void __weak r6k_cache_init(void); 16702cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle 16802cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle r6k_cache_init(); 1691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 17002cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle if (cpu_has_4k_cache) { 17102cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle extern void __weak r4k_cache_init(void); 17202cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle 17302cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle r4k_cache_init(); 17402cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle } 17502cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle if (cpu_has_8k_cache) { 17602cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle extern void __weak r8k_cache_init(void); 17702cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle 17802cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle r8k_cache_init(); 17902cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle } 18002cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle if (cpu_has_tx39_cache) { 18102cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle extern void __weak tx39_cache_init(void); 18202cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle 18302cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle tx39_cache_init(); 18402cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle } 18502cf2119684e52e97a8a90bd7630386e0f1a250aRalf Baechle 18647d979eca33f8df49bfead2d5efa23a70b413882David Daney if (cpu_has_octeon_cache) { 18747d979eca33f8df49bfead2d5efa23a70b413882David Daney extern void __weak octeon_cache_init(void); 18847d979eca33f8df49bfead2d5efa23a70b413882David Daney 18947d979eca33f8df49bfead2d5efa23a70b413882David Daney octeon_cache_init(); 19047d979eca33f8df49bfead2d5efa23a70b413882David Daney } 19147d979eca33f8df49bfead2d5efa23a70b413882David Daney 192351336929ccf222ae38ff0cb7a8dd5fd5c6236a0Chris Dearman setup_protection_map(); 1931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 19424e9d0b96dac5503c0b6f034d553030c604228a7Ralf Baechle 19524e9d0b96dac5503c0b6f034d553030c604228a7Ralf Baechleint __weak __uncached_access(struct file *file, unsigned long addr) 19624e9d0b96dac5503c0b6f034d553030c604228a7Ralf Baechle{ 1976b2f3d1f769be5779b479c37800229d9a4809fc3Christoph Hellwig if (file->f_flags & O_DSYNC) 19824e9d0b96dac5503c0b6f034d553030c604228a7Ralf Baechle return 1; 19924e9d0b96dac5503c0b6f034d553030c604228a7Ralf Baechle 20024e9d0b96dac5503c0b6f034d553030c604228a7Ralf Baechle return addr >= __pa(high_memory); 20124e9d0b96dac5503c0b6f034d553030c604228a7Ralf Baechle} 202