117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen/* 217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Copyright (C) 2001-2006 Silicon Graphics, Inc. All rights 317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * reserved. 417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * 517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * This program is free software; you can redistribute it and/or modify it 617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * under the terms of version 2 of the GNU General Public License 717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * as published by the Free Software Foundation. 817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen */ 917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 1017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen/* 1117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * SN Platform Special Memory (mspec) Support 1217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * 1317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * This driver exports the SN special memory (mspec) facility to user 1417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * processes. 1517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * There are three types of memory made available thru this driver: 1617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * fetchops, uncached and cached. 1717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * 1817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Fetchops are atomic memory operations that are implemented in the 1917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * memory controller on SGI SN hardware. 2017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * 2117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Uncached are used for memory write combining feature of the ia64 2217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * cpu. 2317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * 2417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Cached are used for areas of memory that are used as cached addresses 2517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * on our partition and used as uncached addresses from other partitions. 2617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Due to a design constraint of the SN2 Shub, you can not have processors 2717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * on the same FSB perform both a cached and uncached reference to the 2817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * same cache line. These special memory cached regions prevent the 2917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * kernel from ever dropping in a TLB entry and therefore prevent the 3017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * processor from ever speculating a cache line from this page. 3117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen */ 3217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 3317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/types.h> 3417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/kernel.h> 3517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/module.h> 3617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/init.h> 3717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/errno.h> 3817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/miscdevice.h> 3917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/spinlock.h> 4017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/mm.h> 414e950f6f0189f65f8bf069cf2272649ef418f5e4Alexey Dobriyan#include <linux/fs.h> 4217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/vmalloc.h> 4317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/string.h> 4417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/slab.h> 4517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/numa.h> 4617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/page.h> 4717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/pgtable.h> 4860063497a95e716c9a689af3be2687d261f115b4Arun Sharma#include <linux/atomic.h> 4917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/tlbflush.h> 5017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/uncached.h> 5117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/sn/addrs.h> 5217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/sn/arch.h> 5317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/sn/mspec.h> 5417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/sn/sn_cpuid.h> 5517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/sn/io.h> 5617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/sn/bte.h> 5717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/sn/shubio.h> 5817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 5917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 6017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#define FETCHOP_ID "SGI Fetchop," 6117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#define CACHED_ID "Cached," 6217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#define UNCACHED_ID "Uncached" 6317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#define REVISION "4.0" 6417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#define MSPEC_BASENAME "mspec" 6517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 6617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen/* 6717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Page types allocated by the device. 6817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen */ 694191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickmanenum mspec_page_type { 7017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen MSPEC_FETCHOP = 1, 7117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen MSPEC_CACHED, 7217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen MSPEC_UNCACHED 7317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen}; 7417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 751a4b0fc503ff4149f5915be4aeb179b9453cf485Jes Sorensen#ifdef CONFIG_SGI_SN 7617a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic int is_sn2; 771a4b0fc503ff4149f5915be4aeb179b9453cf485Jes Sorensen#else 781a4b0fc503ff4149f5915be4aeb179b9453cf485Jes Sorensen#define is_sn2 0 791a4b0fc503ff4149f5915be4aeb179b9453cf485Jes Sorensen#endif 8017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 8117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen/* 8217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * One of these structures is allocated when an mspec region is mmaped. The 8317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * structure is pointed to by the vma->vm_private_data field in the vma struct. 8417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * This structure is used to record the addresses of the mspec pages. 854191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman * This structure is shared by all vma's that are split off from the 864191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman * original vma when split_vma()'s are done. 874191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman * 884191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman * The refcnt is incremented atomically because mm->mmap_sem does not 894191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman * protect in fork case where multiple tasks share the vma_data. 9017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen */ 9117a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstruct vma_data { 9217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen atomic_t refcnt; /* Number of vmas sharing the data. */ 934191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman spinlock_t lock; /* Serialize access to this structure. */ 9417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen int count; /* Number of pages allocated. */ 954191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman enum mspec_page_type type; /* Type of pages allocated. */ 964191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman int flags; /* See VMD_xxx below. */ 974191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman unsigned long vm_start; /* Original (unsplit) base. */ 984191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman unsigned long vm_end; /* Original (unsplit) end. */ 9917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen unsigned long maddr[0]; /* Array of MSPEC addresses. */ 10017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen}; 10117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 1024191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman#define VMD_VMALLOCED 0x1 /* vmalloc'd rather than kmalloc'd */ 1034191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman 10417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen/* used on shub2 to clear FOP cache in the HUB */ 10517a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic unsigned long scratch_page[MAX_NUMNODES]; 10617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#define SH2_AMO_CACHE_ENTRIES 4 10717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 10817a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic inline int 10917a3b05047119b7fc72ef03962e202becc659579Jes Sorensenmspec_zero_block(unsigned long addr, int len) 11017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen{ 11117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen int status; 11217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 11317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (is_sn2) { 11417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (is_shub2()) { 11517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen int nid; 11617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen void *p; 11717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen int i; 11817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 11917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen nid = nasid_to_cnodeid(get_node_number(__pa(addr))); 12017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen p = (void *)TO_AMO(scratch_page[nid]); 12117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 12217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen for (i=0; i < SH2_AMO_CACHE_ENTRIES; i++) { 12317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen FETCHOP_LOAD_OP(p, FETCHOP_LOAD); 12417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen p += FETCHOP_VAR_SIZE; 12517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 12617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 12717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 12817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen status = bte_copy(0, addr & ~__IA64_UNCACHED_OFFSET, len, 12917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen BTE_WACQUIRE | BTE_ZERO_FILL, NULL); 13017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } else { 13117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen memset((char *) addr, 0, len); 13217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen status = 0; 13317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 13417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return status; 13517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen} 13617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 13717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen/* 13817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * mspec_open 13917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * 14017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Called when a device mapping is created by a means other than mmap 1414191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman * (via fork, munmap, etc.). Increments the reference count on the 1424191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman * underlying mspec data so it is not freed prematurely. 14317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen */ 14417a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic void 14517a3b05047119b7fc72ef03962e202becc659579Jes Sorensenmspec_open(struct vm_area_struct *vma) 14617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen{ 14717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen struct vma_data *vdata; 14817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 14917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vdata = vma->vm_private_data; 15017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen atomic_inc(&vdata->refcnt); 15117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen} 15217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 15317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen/* 15417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * mspec_close 15517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * 15617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Called when unmapping a device mapping. Frees all mspec pages 157afa684f6fda6086b229348f0ea21df7c8ad17964Cliff Wickman * belonging to all the vma's sharing this vma_data structure. 15817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen */ 15917a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic void 16017a3b05047119b7fc72ef03962e202becc659579Jes Sorensenmspec_close(struct vm_area_struct *vma) 16117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen{ 16217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen struct vma_data *vdata; 163afa684f6fda6086b229348f0ea21df7c8ad17964Cliff Wickman int index, last_index; 1644191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman unsigned long my_page; 16517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 16617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vdata = vma->vm_private_data; 16717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 168afa684f6fda6086b229348f0ea21df7c8ad17964Cliff Wickman if (!atomic_dec_and_test(&vdata->refcnt)) 169afa684f6fda6086b229348f0ea21df7c8ad17964Cliff Wickman return; 1704191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman 171afa684f6fda6086b229348f0ea21df7c8ad17964Cliff Wickman last_index = (vdata->vm_end - vdata->vm_start) >> PAGE_SHIFT; 172afa684f6fda6086b229348f0ea21df7c8ad17964Cliff Wickman for (index = 0; index < last_index; index++) { 1734191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman if (vdata->maddr[index] == 0) 17417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen continue; 17517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen /* 17617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Clear the page before sticking it back 17717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * into the pool. 17817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen */ 1794191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman my_page = vdata->maddr[index]; 1804191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman vdata->maddr[index] = 0; 181afa684f6fda6086b229348f0ea21df7c8ad17964Cliff Wickman if (!mspec_zero_block(my_page, PAGE_SIZE)) 182e4a064dfa2b242519a9f06f9a1e58c27bf0c371bDean Nelson uncached_free_page(my_page, 1); 18317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen else 18417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen printk(KERN_WARNING "mspec_close(): " 185afa684f6fda6086b229348f0ea21df7c8ad17964Cliff Wickman "failed to zero page %ld\n", my_page); 18617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 1874191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman 1884191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman if (vdata->flags & VMD_VMALLOCED) 18917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vfree(vdata); 1904191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman else 1914191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman kfree(vdata); 19217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen} 19317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 19417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen/* 195efe9e77997f6e0306fedc6efa98df491dcf5ecb0Nick Piggin * mspec_fault 19617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * 19717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Creates a mspec page and maps it to user space. 19817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen */ 199efe9e77997f6e0306fedc6efa98df491dcf5ecb0Nick Pigginstatic int 200efe9e77997f6e0306fedc6efa98df491dcf5ecb0Nick Pigginmspec_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 20117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen{ 20217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen unsigned long paddr, maddr; 20317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen unsigned long pfn; 204efe9e77997f6e0306fedc6efa98df491dcf5ecb0Nick Piggin pgoff_t index = vmf->pgoff; 20517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen struct vma_data *vdata = vma->vm_private_data; 20617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 20717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen maddr = (volatile unsigned long) vdata->maddr[index]; 20817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (maddr == 0) { 209e4a064dfa2b242519a9f06f9a1e58c27bf0c371bDean Nelson maddr = uncached_alloc_page(numa_node_id(), 1); 21017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (maddr == 0) 211efe9e77997f6e0306fedc6efa98df491dcf5ecb0Nick Piggin return VM_FAULT_OOM; 21217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 21317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen spin_lock(&vdata->lock); 21417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (vdata->maddr[index] == 0) { 21517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vdata->count++; 21617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vdata->maddr[index] = maddr; 21717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } else { 218e4a064dfa2b242519a9f06f9a1e58c27bf0c371bDean Nelson uncached_free_page(maddr, 1); 21917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen maddr = vdata->maddr[index]; 22017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 22117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen spin_unlock(&vdata->lock); 22217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 22317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 22417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (vdata->type == MSPEC_FETCHOP) 22517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen paddr = TO_AMO(maddr); 22617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen else 2271a4b0fc503ff4149f5915be4aeb179b9453cf485Jes Sorensen paddr = maddr & ~__IA64_UNCACHED_OFFSET; 22817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 22917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen pfn = paddr >> PAGE_SHIFT; 23017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 231efe9e77997f6e0306fedc6efa98df491dcf5ecb0Nick Piggin /* 232efe9e77997f6e0306fedc6efa98df491dcf5ecb0Nick Piggin * vm_insert_pfn can fail with -EBUSY, but in that case it will 233efe9e77997f6e0306fedc6efa98df491dcf5ecb0Nick Piggin * be because another thread has installed the pte first, so it 234efe9e77997f6e0306fedc6efa98df491dcf5ecb0Nick Piggin * is no problem. 235efe9e77997f6e0306fedc6efa98df491dcf5ecb0Nick Piggin */ 236efe9e77997f6e0306fedc6efa98df491dcf5ecb0Nick Piggin vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn); 237efe9e77997f6e0306fedc6efa98df491dcf5ecb0Nick Piggin 238efe9e77997f6e0306fedc6efa98df491dcf5ecb0Nick Piggin return VM_FAULT_NOPAGE; 23917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen} 24017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 241f0f37e2f77731b3473fa6bd5ee53255d9a9cdb40Alexey Dobriyanstatic const struct vm_operations_struct mspec_vm_ops = { 24217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .open = mspec_open, 24317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .close = mspec_close, 244efe9e77997f6e0306fedc6efa98df491dcf5ecb0Nick Piggin .fault = mspec_fault, 24517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen}; 24617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 24717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen/* 24817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * mspec_mmap 24917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * 250af901ca181d92aac3a7dc265144a9081a86d8f39André Goddard Rosa * Called when mmapping the device. Initializes the vma with a fault handler 25117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * and private data structure necessary to allocate, track, and free the 25217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * underlying pages. 25317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen */ 25417a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic int 2554191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickmanmspec_mmap(struct file *file, struct vm_area_struct *vma, 2564191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman enum mspec_page_type type) 25717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen{ 25817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen struct vma_data *vdata; 2594191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman int pages, vdata_size, flags = 0; 26017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 26117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (vma->vm_pgoff != 0) 26217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return -EINVAL; 26317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 26417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if ((vma->vm_flags & VM_SHARED) == 0) 26517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return -EINVAL; 26617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 26717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if ((vma->vm_flags & VM_WRITE) == 0) 26817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return -EPERM; 26917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 270a0ea59d56dfab021ecc65365275e532c6b937adbLibin pages = vma_pages(vma); 27117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vdata_size = sizeof(struct vma_data) + pages * sizeof(long); 27217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (vdata_size <= PAGE_SIZE) 273658c74cf3c98b1c9bc21e26731052db66251dfd8Rakib Mullick vdata = kzalloc(vdata_size, GFP_KERNEL); 2744191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman else { 275658c74cf3c98b1c9bc21e26731052db66251dfd8Rakib Mullick vdata = vzalloc(vdata_size); 2764191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman flags = VMD_VMALLOCED; 2774191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman } 27817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (!vdata) 27917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return -ENOMEM; 28017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 2814191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman vdata->vm_start = vma->vm_start; 2824191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman vdata->vm_end = vma->vm_end; 2834191ba26dae8338892e73f6e67bd18068b4344e9Cliff Wickman vdata->flags = flags; 28417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vdata->type = type; 28517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen spin_lock_init(&vdata->lock); 286a119365586b0130dfea06457f584953e0ff6481dTony Luck atomic_set(&vdata->refcnt, 1); 28717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vma->vm_private_data = vdata; 28817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 289314e51b9851b4f4e8ab302243ff5a6fc6147f379Konstantin Khlebnikov vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; 29017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (vdata->type == MSPEC_FETCHOP || vdata->type == MSPEC_UNCACHED) 29117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 29217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vma->vm_ops = &mspec_vm_ops; 29317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 29417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return 0; 29517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen} 29617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 29717a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic int 29817a3b05047119b7fc72ef03962e202becc659579Jes Sorensenfetchop_mmap(struct file *file, struct vm_area_struct *vma) 29917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen{ 30017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return mspec_mmap(file, vma, MSPEC_FETCHOP); 30117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen} 30217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 30317a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic int 30417a3b05047119b7fc72ef03962e202becc659579Jes Sorensencached_mmap(struct file *file, struct vm_area_struct *vma) 30517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen{ 30617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return mspec_mmap(file, vma, MSPEC_CACHED); 30717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen} 30817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 30917a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic int 31017a3b05047119b7fc72ef03962e202becc659579Jes Sorensenuncached_mmap(struct file *file, struct vm_area_struct *vma) 31117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen{ 31217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return mspec_mmap(file, vma, MSPEC_UNCACHED); 31317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen} 31417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 3152b8693c0617e972fc0b2fd1ebf8de97e15b656c3Arjan van de Venstatic const struct file_operations fetchop_fops = { 31617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .owner = THIS_MODULE, 3176038f373a3dc1f1c26496e60b6c40b164716f07eArnd Bergmann .mmap = fetchop_mmap, 3186038f373a3dc1f1c26496e60b6c40b164716f07eArnd Bergmann .llseek = noop_llseek, 31917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen}; 32017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 32117a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic struct miscdevice fetchop_miscdev = { 32217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .minor = MISC_DYNAMIC_MINOR, 32317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .name = "sgi_fetchop", 32417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .fops = &fetchop_fops 32517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen}; 32617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 3272b8693c0617e972fc0b2fd1ebf8de97e15b656c3Arjan van de Venstatic const struct file_operations cached_fops = { 32817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .owner = THIS_MODULE, 3296038f373a3dc1f1c26496e60b6c40b164716f07eArnd Bergmann .mmap = cached_mmap, 3306038f373a3dc1f1c26496e60b6c40b164716f07eArnd Bergmann .llseek = noop_llseek, 33117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen}; 33217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 33317a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic struct miscdevice cached_miscdev = { 33417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .minor = MISC_DYNAMIC_MINOR, 33517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .name = "mspec_cached", 33617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .fops = &cached_fops 33717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen}; 33817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 3392b8693c0617e972fc0b2fd1ebf8de97e15b656c3Arjan van de Venstatic const struct file_operations uncached_fops = { 34017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .owner = THIS_MODULE, 3416038f373a3dc1f1c26496e60b6c40b164716f07eArnd Bergmann .mmap = uncached_mmap, 3426038f373a3dc1f1c26496e60b6c40b164716f07eArnd Bergmann .llseek = noop_llseek, 34317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen}; 34417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 34517a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic struct miscdevice uncached_miscdev = { 34617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .minor = MISC_DYNAMIC_MINOR, 34717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .name = "mspec_uncached", 34817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .fops = &uncached_fops 34917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen}; 35017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 35117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen/* 35217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * mspec_init 35317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * 35417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Called at boot time to initialize the mspec facility. 35517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen */ 35617a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic int __init 35717a3b05047119b7fc72ef03962e202becc659579Jes Sorensenmspec_init(void) 35817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen{ 35917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen int ret; 36017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen int nid; 36117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 36217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen /* 36317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * The fetchop device only works on SN2 hardware, uncached and cached 36417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * memory drivers should both be valid on all ia64 hardware 36517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen */ 3661a4b0fc503ff4149f5915be4aeb179b9453cf485Jes Sorensen#ifdef CONFIG_SGI_SN 36717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (ia64_platform_is("sn2")) { 36817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen is_sn2 = 1; 36917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (is_shub2()) { 37017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen ret = -ENOMEM; 3712dca53a9dabe76f49209c9128313347510416c68Christoph Lameter for_each_node_state(nid, N_ONLINE) { 37217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen int actual_nid; 37317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen int nasid; 37417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen unsigned long phys; 37517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 376e4a064dfa2b242519a9f06f9a1e58c27bf0c371bDean Nelson scratch_page[nid] = uncached_alloc_page(nid, 1); 37717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (scratch_page[nid] == 0) 37817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen goto free_scratch_pages; 37917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen phys = __pa(scratch_page[nid]); 38017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen nasid = get_node_number(phys); 38117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen actual_nid = nasid_to_cnodeid(nasid); 38217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (actual_nid != nid) 38317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen goto free_scratch_pages; 38417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 38517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 38617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 38717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen ret = misc_register(&fetchop_miscdev); 38817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (ret) { 38917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen printk(KERN_ERR 39017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen "%s: failed to register device %i\n", 39117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen FETCHOP_ID, ret); 39217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen goto free_scratch_pages; 39317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 39417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 3951a4b0fc503ff4149f5915be4aeb179b9453cf485Jes Sorensen#endif 39617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen ret = misc_register(&cached_miscdev); 39717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (ret) { 39817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen printk(KERN_ERR "%s: failed to register device %i\n", 39917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen CACHED_ID, ret); 40017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (is_sn2) 40117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen misc_deregister(&fetchop_miscdev); 40217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen goto free_scratch_pages; 40317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 40417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen ret = misc_register(&uncached_miscdev); 40517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (ret) { 40617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen printk(KERN_ERR "%s: failed to register device %i\n", 40717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen UNCACHED_ID, ret); 40817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen misc_deregister(&cached_miscdev); 40917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (is_sn2) 41017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen misc_deregister(&fetchop_miscdev); 41117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen goto free_scratch_pages; 41217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 41317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 41417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen printk(KERN_INFO "%s %s initialized devices: %s %s %s\n", 41517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen MSPEC_BASENAME, REVISION, is_sn2 ? FETCHOP_ID : "", 41617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen CACHED_ID, UNCACHED_ID); 41717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 41817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return 0; 41917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 42017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen free_scratch_pages: 42117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen for_each_node(nid) { 42217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (scratch_page[nid] != 0) 423e4a064dfa2b242519a9f06f9a1e58c27bf0c371bDean Nelson uncached_free_page(scratch_page[nid], 1); 42417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 42517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return ret; 42617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen} 42717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 42817a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic void __exit 42917a3b05047119b7fc72ef03962e202becc659579Jes Sorensenmspec_exit(void) 43017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen{ 43117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen int nid; 43217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 43317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen misc_deregister(&uncached_miscdev); 43417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen misc_deregister(&cached_miscdev); 43517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (is_sn2) { 43617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen misc_deregister(&fetchop_miscdev); 43717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 43817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen for_each_node(nid) { 43917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (scratch_page[nid] != 0) 440e4a064dfa2b242519a9f06f9a1e58c27bf0c371bDean Nelson uncached_free_page(scratch_page[nid], 1); 44117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 44217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 44317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen} 44417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 44517a3b05047119b7fc72ef03962e202becc659579Jes Sorensenmodule_init(mspec_init); 44617a3b05047119b7fc72ef03962e202becc659579Jes Sorensenmodule_exit(mspec_exit); 44717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 44817a3b05047119b7fc72ef03962e202becc659579Jes SorensenMODULE_AUTHOR("Silicon Graphics, Inc. <linux-altix@sgi.com>"); 44917a3b05047119b7fc72ef03962e202becc659579Jes SorensenMODULE_DESCRIPTION("Driver for SGI SN special memory operations"); 45017a3b05047119b7fc72ef03962e202becc659579Jes SorensenMODULE_LICENSE("GPL"); 451