mspec.c revision 2b8693c0617e972fc0b2fd1ebf8de97e15b656c3
117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen/* 217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Copyright (C) 2001-2006 Silicon Graphics, Inc. All rights 317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * reserved. 417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * 517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * This program is free software; you can redistribute it and/or modify it 617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * under the terms of version 2 of the GNU General Public License 717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * as published by the Free Software Foundation. 817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen */ 917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 1017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen/* 1117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * SN Platform Special Memory (mspec) Support 1217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * 1317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * This driver exports the SN special memory (mspec) facility to user 1417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * processes. 1517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * There are three types of memory made available thru this driver: 1617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * fetchops, uncached and cached. 1717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * 1817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Fetchops are atomic memory operations that are implemented in the 1917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * memory controller on SGI SN hardware. 2017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * 2117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Uncached are used for memory write combining feature of the ia64 2217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * cpu. 2317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * 2417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Cached are used for areas of memory that are used as cached addresses 2517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * on our partition and used as uncached addresses from other partitions. 2617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Due to a design constraint of the SN2 Shub, you can not have processors 2717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * on the same FSB perform both a cached and uncached reference to the 2817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * same cache line. These special memory cached regions prevent the 2917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * kernel from ever dropping in a TLB entry and therefore prevent the 3017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * processor from ever speculating a cache line from this page. 3117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen */ 3217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 3317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/types.h> 3417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/kernel.h> 3517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/module.h> 3617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/init.h> 3717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/errno.h> 3817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/miscdevice.h> 3917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/spinlock.h> 4017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/mm.h> 4117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/vmalloc.h> 4217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/string.h> 4317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/slab.h> 4417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <linux/numa.h> 4517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/page.h> 4617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/system.h> 4717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/pgtable.h> 4817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/atomic.h> 4917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/tlbflush.h> 5017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/uncached.h> 5117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/sn/addrs.h> 5217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/sn/arch.h> 5317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/sn/mspec.h> 5417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/sn/sn_cpuid.h> 5517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/sn/io.h> 5617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/sn/bte.h> 5717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#include <asm/sn/shubio.h> 5817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 5917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 6017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#define FETCHOP_ID "SGI Fetchop," 6117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#define CACHED_ID "Cached," 6217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#define UNCACHED_ID "Uncached" 6317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#define REVISION "4.0" 6417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#define MSPEC_BASENAME "mspec" 6517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 6617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen/* 6717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Page types allocated by the device. 6817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen */ 6917a3b05047119b7fc72ef03962e202becc659579Jes Sorensenenum { 7017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen MSPEC_FETCHOP = 1, 7117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen MSPEC_CACHED, 7217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen MSPEC_UNCACHED 7317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen}; 7417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 751a4b0fc503ff4149f5915be4aeb179b9453cf485Jes Sorensen#ifdef CONFIG_SGI_SN 7617a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic int is_sn2; 771a4b0fc503ff4149f5915be4aeb179b9453cf485Jes Sorensen#else 781a4b0fc503ff4149f5915be4aeb179b9453cf485Jes Sorensen#define is_sn2 0 791a4b0fc503ff4149f5915be4aeb179b9453cf485Jes Sorensen#endif 8017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 8117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen/* 8217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * One of these structures is allocated when an mspec region is mmaped. The 8317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * structure is pointed to by the vma->vm_private_data field in the vma struct. 8417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * This structure is used to record the addresses of the mspec pages. 8517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen */ 8617a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstruct vma_data { 8717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen atomic_t refcnt; /* Number of vmas sharing the data. */ 8817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen spinlock_t lock; /* Serialize access to the vma. */ 8917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen int count; /* Number of pages allocated. */ 9017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen int type; /* Type of pages allocated. */ 9117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen unsigned long maddr[0]; /* Array of MSPEC addresses. */ 9217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen}; 9317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 9417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen/* used on shub2 to clear FOP cache in the HUB */ 9517a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic unsigned long scratch_page[MAX_NUMNODES]; 9617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen#define SH2_AMO_CACHE_ENTRIES 4 9717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 9817a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic inline int 9917a3b05047119b7fc72ef03962e202becc659579Jes Sorensenmspec_zero_block(unsigned long addr, int len) 10017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen{ 10117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen int status; 10217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 10317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (is_sn2) { 10417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (is_shub2()) { 10517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen int nid; 10617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen void *p; 10717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen int i; 10817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 10917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen nid = nasid_to_cnodeid(get_node_number(__pa(addr))); 11017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen p = (void *)TO_AMO(scratch_page[nid]); 11117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 11217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen for (i=0; i < SH2_AMO_CACHE_ENTRIES; i++) { 11317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen FETCHOP_LOAD_OP(p, FETCHOP_LOAD); 11417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen p += FETCHOP_VAR_SIZE; 11517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 11617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 11717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 11817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen status = bte_copy(0, addr & ~__IA64_UNCACHED_OFFSET, len, 11917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen BTE_WACQUIRE | BTE_ZERO_FILL, NULL); 12017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } else { 12117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen memset((char *) addr, 0, len); 12217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen status = 0; 12317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 12417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return status; 12517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen} 12617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 12717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen/* 12817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * mspec_open 12917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * 13017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Called when a device mapping is created by a means other than mmap 13117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * (via fork, etc.). Increments the reference count on the underlying 13217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * mspec data so it is not freed prematurely. 13317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen */ 13417a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic void 13517a3b05047119b7fc72ef03962e202becc659579Jes Sorensenmspec_open(struct vm_area_struct *vma) 13617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen{ 13717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen struct vma_data *vdata; 13817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 13917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vdata = vma->vm_private_data; 14017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen atomic_inc(&vdata->refcnt); 14117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen} 14217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 14317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen/* 14417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * mspec_close 14517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * 14617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Called when unmapping a device mapping. Frees all mspec pages 14717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * belonging to the vma. 14817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen */ 14917a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic void 15017a3b05047119b7fc72ef03962e202becc659579Jes Sorensenmspec_close(struct vm_area_struct *vma) 15117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen{ 15217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen struct vma_data *vdata; 15317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen int i, pages, result, vdata_size; 15417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 15517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vdata = vma->vm_private_data; 15617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (!atomic_dec_and_test(&vdata->refcnt)) 15717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return; 15817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 15917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; 16017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vdata_size = sizeof(struct vma_data) + pages * sizeof(long); 16117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen for (i = 0; i < pages; i++) { 16217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (vdata->maddr[i] == 0) 16317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen continue; 16417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen /* 16517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Clear the page before sticking it back 16617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * into the pool. 16717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen */ 16817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen result = mspec_zero_block(vdata->maddr[i], PAGE_SIZE); 16917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (!result) 17017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen uncached_free_page(vdata->maddr[i]); 17117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen else 17217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen printk(KERN_WARNING "mspec_close(): " 17317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen "failed to zero page %i\n", 17417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen result); 17517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 17617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 17717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (vdata_size <= PAGE_SIZE) 17817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen kfree(vdata); 17917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen else 18017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vfree(vdata); 18117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen} 18217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 18317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 18417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen/* 18517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * mspec_nopfn 18617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * 18717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Creates a mspec page and maps it to user space. 18817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen */ 18917a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic unsigned long 19017a3b05047119b7fc72ef03962e202becc659579Jes Sorensenmspec_nopfn(struct vm_area_struct *vma, unsigned long address) 19117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen{ 19217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen unsigned long paddr, maddr; 19317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen unsigned long pfn; 19417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen int index; 19517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen struct vma_data *vdata = vma->vm_private_data; 19617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 19717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen index = (address - vma->vm_start) >> PAGE_SHIFT; 19817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen maddr = (volatile unsigned long) vdata->maddr[index]; 19917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (maddr == 0) { 20017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen maddr = uncached_alloc_page(numa_node_id()); 20117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (maddr == 0) 20217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return NOPFN_OOM; 20317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 20417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen spin_lock(&vdata->lock); 20517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (vdata->maddr[index] == 0) { 20617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vdata->count++; 20717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vdata->maddr[index] = maddr; 20817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } else { 20917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen uncached_free_page(maddr); 21017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen maddr = vdata->maddr[index]; 21117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 21217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen spin_unlock(&vdata->lock); 21317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 21417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 21517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (vdata->type == MSPEC_FETCHOP) 21617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen paddr = TO_AMO(maddr); 21717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen else 2181a4b0fc503ff4149f5915be4aeb179b9453cf485Jes Sorensen paddr = maddr & ~__IA64_UNCACHED_OFFSET; 21917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 22017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen pfn = paddr >> PAGE_SHIFT; 22117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 22217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return pfn; 22317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen} 22417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 22517a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic struct vm_operations_struct mspec_vm_ops = { 22617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .open = mspec_open, 22717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .close = mspec_close, 22817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .nopfn = mspec_nopfn 22917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen}; 23017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 23117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen/* 23217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * mspec_mmap 23317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * 23417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Called when mmaping the device. Initializes the vma with a fault handler 23517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * and private data structure necessary to allocate, track, and free the 23617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * underlying pages. 23717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen */ 23817a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic int 23917a3b05047119b7fc72ef03962e202becc659579Jes Sorensenmspec_mmap(struct file *file, struct vm_area_struct *vma, int type) 24017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen{ 24117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen struct vma_data *vdata; 24217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen int pages, vdata_size; 24317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 24417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (vma->vm_pgoff != 0) 24517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return -EINVAL; 24617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 24717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if ((vma->vm_flags & VM_SHARED) == 0) 24817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return -EINVAL; 24917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 25017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if ((vma->vm_flags & VM_WRITE) == 0) 25117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return -EPERM; 25217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 25317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; 25417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vdata_size = sizeof(struct vma_data) + pages * sizeof(long); 25517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (vdata_size <= PAGE_SIZE) 25617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vdata = kmalloc(vdata_size, GFP_KERNEL); 25717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen else 25817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vdata = vmalloc(vdata_size); 25917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (!vdata) 26017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return -ENOMEM; 26117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen memset(vdata, 0, vdata_size); 26217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 26317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vdata->type = type; 26417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen spin_lock_init(&vdata->lock); 26517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vdata->refcnt = ATOMIC_INIT(1); 26617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vma->vm_private_data = vdata; 26717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 26817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vma->vm_flags |= (VM_IO | VM_LOCKED | VM_RESERVED | VM_PFNMAP); 26917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (vdata->type == MSPEC_FETCHOP || vdata->type == MSPEC_UNCACHED) 27017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 27117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen vma->vm_ops = &mspec_vm_ops; 27217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 27317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return 0; 27417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen} 27517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 27617a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic int 27717a3b05047119b7fc72ef03962e202becc659579Jes Sorensenfetchop_mmap(struct file *file, struct vm_area_struct *vma) 27817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen{ 27917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return mspec_mmap(file, vma, MSPEC_FETCHOP); 28017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen} 28117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 28217a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic int 28317a3b05047119b7fc72ef03962e202becc659579Jes Sorensencached_mmap(struct file *file, struct vm_area_struct *vma) 28417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen{ 28517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return mspec_mmap(file, vma, MSPEC_CACHED); 28617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen} 28717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 28817a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic int 28917a3b05047119b7fc72ef03962e202becc659579Jes Sorensenuncached_mmap(struct file *file, struct vm_area_struct *vma) 29017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen{ 29117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return mspec_mmap(file, vma, MSPEC_UNCACHED); 29217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen} 29317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 2942b8693c0617e972fc0b2fd1ebf8de97e15b656c3Arjan van de Venstatic const struct file_operations fetchop_fops = { 29517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .owner = THIS_MODULE, 29617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .mmap = fetchop_mmap 29717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen}; 29817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 29917a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic struct miscdevice fetchop_miscdev = { 30017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .minor = MISC_DYNAMIC_MINOR, 30117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .name = "sgi_fetchop", 30217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .fops = &fetchop_fops 30317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen}; 30417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 3052b8693c0617e972fc0b2fd1ebf8de97e15b656c3Arjan van de Venstatic const struct file_operations cached_fops = { 30617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .owner = THIS_MODULE, 30717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .mmap = cached_mmap 30817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen}; 30917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 31017a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic struct miscdevice cached_miscdev = { 31117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .minor = MISC_DYNAMIC_MINOR, 31217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .name = "mspec_cached", 31317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .fops = &cached_fops 31417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen}; 31517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 3162b8693c0617e972fc0b2fd1ebf8de97e15b656c3Arjan van de Venstatic const struct file_operations uncached_fops = { 31717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .owner = THIS_MODULE, 31817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .mmap = uncached_mmap 31917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen}; 32017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 32117a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic struct miscdevice uncached_miscdev = { 32217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .minor = MISC_DYNAMIC_MINOR, 32317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .name = "mspec_uncached", 32417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen .fops = &uncached_fops 32517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen}; 32617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 32717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen/* 32817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * mspec_init 32917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * 33017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * Called at boot time to initialize the mspec facility. 33117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen */ 33217a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic int __init 33317a3b05047119b7fc72ef03962e202becc659579Jes Sorensenmspec_init(void) 33417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen{ 33517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen int ret; 33617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen int nid; 33717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 33817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen /* 33917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * The fetchop device only works on SN2 hardware, uncached and cached 34017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen * memory drivers should both be valid on all ia64 hardware 34117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen */ 3421a4b0fc503ff4149f5915be4aeb179b9453cf485Jes Sorensen#ifdef CONFIG_SGI_SN 34317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (ia64_platform_is("sn2")) { 34417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen is_sn2 = 1; 34517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (is_shub2()) { 34617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen ret = -ENOMEM; 34717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen for_each_online_node(nid) { 34817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen int actual_nid; 34917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen int nasid; 35017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen unsigned long phys; 35117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 35217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen scratch_page[nid] = uncached_alloc_page(nid); 35317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (scratch_page[nid] == 0) 35417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen goto free_scratch_pages; 35517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen phys = __pa(scratch_page[nid]); 35617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen nasid = get_node_number(phys); 35717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen actual_nid = nasid_to_cnodeid(nasid); 35817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (actual_nid != nid) 35917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen goto free_scratch_pages; 36017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 36117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 36217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 36317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen ret = misc_register(&fetchop_miscdev); 36417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (ret) { 36517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen printk(KERN_ERR 36617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen "%s: failed to register device %i\n", 36717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen FETCHOP_ID, ret); 36817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen goto free_scratch_pages; 36917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 37017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 3711a4b0fc503ff4149f5915be4aeb179b9453cf485Jes Sorensen#endif 37217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen ret = misc_register(&cached_miscdev); 37317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (ret) { 37417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen printk(KERN_ERR "%s: failed to register device %i\n", 37517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen CACHED_ID, ret); 37617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (is_sn2) 37717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen misc_deregister(&fetchop_miscdev); 37817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen goto free_scratch_pages; 37917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 38017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen ret = misc_register(&uncached_miscdev); 38117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (ret) { 38217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen printk(KERN_ERR "%s: failed to register device %i\n", 38317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen UNCACHED_ID, ret); 38417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen misc_deregister(&cached_miscdev); 38517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (is_sn2) 38617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen misc_deregister(&fetchop_miscdev); 38717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen goto free_scratch_pages; 38817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 38917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 39017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen printk(KERN_INFO "%s %s initialized devices: %s %s %s\n", 39117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen MSPEC_BASENAME, REVISION, is_sn2 ? FETCHOP_ID : "", 39217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen CACHED_ID, UNCACHED_ID); 39317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 39417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return 0; 39517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 39617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen free_scratch_pages: 39717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen for_each_node(nid) { 39817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (scratch_page[nid] != 0) 39917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen uncached_free_page(scratch_page[nid]); 40017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 40117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen return ret; 40217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen} 40317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 40417a3b05047119b7fc72ef03962e202becc659579Jes Sorensenstatic void __exit 40517a3b05047119b7fc72ef03962e202becc659579Jes Sorensenmspec_exit(void) 40617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen{ 40717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen int nid; 40817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 40917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen misc_deregister(&uncached_miscdev); 41017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen misc_deregister(&cached_miscdev); 41117a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (is_sn2) { 41217a3b05047119b7fc72ef03962e202becc659579Jes Sorensen misc_deregister(&fetchop_miscdev); 41317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 41417a3b05047119b7fc72ef03962e202becc659579Jes Sorensen for_each_node(nid) { 41517a3b05047119b7fc72ef03962e202becc659579Jes Sorensen if (scratch_page[nid] != 0) 41617a3b05047119b7fc72ef03962e202becc659579Jes Sorensen uncached_free_page(scratch_page[nid]); 41717a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 41817a3b05047119b7fc72ef03962e202becc659579Jes Sorensen } 41917a3b05047119b7fc72ef03962e202becc659579Jes Sorensen} 42017a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 42117a3b05047119b7fc72ef03962e202becc659579Jes Sorensenmodule_init(mspec_init); 42217a3b05047119b7fc72ef03962e202becc659579Jes Sorensenmodule_exit(mspec_exit); 42317a3b05047119b7fc72ef03962e202becc659579Jes Sorensen 42417a3b05047119b7fc72ef03962e202becc659579Jes SorensenMODULE_AUTHOR("Silicon Graphics, Inc. <linux-altix@sgi.com>"); 42517a3b05047119b7fc72ef03962e202becc659579Jes SorensenMODULE_DESCRIPTION("Driver for SGI SN special memory operations"); 42617a3b05047119b7fc72ef03962e202becc659579Jes SorensenMODULE_LICENSE("GPL"); 427