1ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller/* iommu.c: Generic sparc64 IOMMU support. 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 3d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller * Copyright (C) 1999, 2007, 2008 David S. Miller (davem@davemloft.net) 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (C) 1999, 2000 Jakub Jelinek (jakub@redhat.com) 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/kernel.h> 8066bcaca51946c8305e3d637a795e8ccf8dbd3cfPaul Gortmaker#include <linux/export.h> 95a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 104dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller#include <linux/delay.h> 11ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller#include <linux/device.h> 12ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller#include <linux/dma-mapping.h> 13ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller#include <linux/errno.h> 14d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller#include <linux/iommu-helper.h> 15a66022c457755b5eef61e30866114679c95e1f54Akinobu Mita#include <linux/bitmap.h> 16ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller 17ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller#ifdef CONFIG_PCI 18c57c2ffb153a99769a15a2ff1729371ddee5601aDavid S. Miller#include <linux/pci.h> 19ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller#endif 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 21ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller#include <asm/iommu.h> 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include "iommu_common.h" 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 25ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller#define STC_CTXMATCH_ADDR(STC, CTX) \ 261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ((STC)->strbuf_ctxmatch_base + ((CTX) << 3)) 27ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller#define STC_FLUSHFLAG_INIT(STC) \ 28ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller (*((STC)->strbuf_flushflag) = 0UL) 29ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller#define STC_FLUSHFLAG_SET(STC) \ 30ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller (*((STC)->strbuf_flushflag) != 0UL) 311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 32ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller#define iommu_read(__reg) \ 331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds({ u64 __ret; \ 341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __asm__ __volatile__("ldxa [%1] %2, %0" \ 351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds : "=r" (__ret) \ 361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds : "r" (__reg), "i" (ASI_PHYS_BYPASS_EC_E) \ 371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds : "memory"); \ 381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __ret; \ 391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}) 40ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller#define iommu_write(__reg, __val) \ 411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __asm__ __volatile__("stxa %0, [%1] %2" \ 421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds : /* no outputs */ \ 431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds : "r" (__val), "r" (__reg), \ 441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "i" (ASI_PHYS_BYPASS_EC_E)) 451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* Must be invoked under the IOMMU lock. */ 47d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Millerstatic void iommu_flushall(struct iommu *iommu) 481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 49861fe90656b8e20d750d73c57088dc52d316ce7bDavid S. Miller if (iommu->iommu_flushinv) { 50ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller iommu_write(iommu->iommu_flushinv, ~(u64)0); 51861fe90656b8e20d750d73c57088dc52d316ce7bDavid S. Miller } else { 52861fe90656b8e20d750d73c57088dc52d316ce7bDavid S. Miller unsigned long tag; 53861fe90656b8e20d750d73c57088dc52d316ce7bDavid S. Miller int entry; 541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 55ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller tag = iommu->iommu_tags; 56861fe90656b8e20d750d73c57088dc52d316ce7bDavid S. Miller for (entry = 0; entry < 16; entry++) { 57ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller iommu_write(tag, 0); 58861fe90656b8e20d750d73c57088dc52d316ce7bDavid S. Miller tag += 8; 59861fe90656b8e20d750d73c57088dc52d316ce7bDavid S. Miller } 601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 61861fe90656b8e20d750d73c57088dc52d316ce7bDavid S. Miller /* Ensure completion of previous PIO writes. */ 62ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller (void) iommu_read(iommu->write_complete_reg); 63861fe90656b8e20d750d73c57088dc52d316ce7bDavid S. Miller } 641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define IOPTE_CONSISTENT(CTX) \ 671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds (IOPTE_VALID | IOPTE_CACHE | \ 681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds (((CTX) << 47) & IOPTE_CONTEXT)) 691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define IOPTE_STREAMING(CTX) \ 711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds (IOPTE_CONSISTENT(CTX) | IOPTE_STBUF) 721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* Existing mappings are never marked invalid, instead they 741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * are pointed to a dummy page. 751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define IOPTE_IS_DUMMY(iommu, iopte) \ 771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ((iopte_val(*iopte) & IOPTE_PAGE) == (iommu)->dummy_page_pa) 781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7916ce82d846f2e6b652a064f91c5019cfe8682be4David S. Millerstatic inline void iopte_make_dummy(struct iommu *iommu, iopte_t *iopte) 801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long val = iopte_val(*iopte); 821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds val &= ~IOPTE_PAGE; 841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds val |= iommu->dummy_page_pa; 851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds iopte_val(*iopte) = val; 871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 89d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller/* Based almost entirely upon the ppc64 iommu allocator. If you use the 'handle' 90d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller * facility it must all be done in one pass while under the iommu lock. 91d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller * 92d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller * On sun4u platforms, we only flush the IOMMU once every time we've passed 93d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller * over the entire page table doing allocations. Therefore we only ever advance 94d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller * the hint and cannot backtrack it. 95d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller */ 96d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Millerunsigned long iommu_range_alloc(struct device *dev, 97d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller struct iommu *iommu, 98d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller unsigned long npages, 99d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller unsigned long *handle) 100688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller{ 101d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller unsigned long n, end, start, limit, boundary_size; 1029b3627f389c07c5be9c86ac4d472a0d4fd47feacDavid S. Miller struct iommu_arena *arena = &iommu->arena; 103d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller int pass = 0; 104d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller 105d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller /* This allocator was derived from x86_64's bit string search */ 106d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller 107d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller /* Sanity check */ 108d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller if (unlikely(npages == 0)) { 109d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller if (printk_ratelimit()) 110d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller WARN_ON(1); 111d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller return DMA_ERROR_CODE; 112d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller } 113d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller 114d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller if (handle && *handle) 115d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller start = *handle; 116d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller else 117d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller start = arena->hint; 118688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller 119688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller limit = arena->limit; 120688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller 121d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller /* The case below can happen if we have a small segment appended 122d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller * to a large, or when the previous alloc was at the very end of 123d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller * the available space. If so, go back to the beginning and flush. 124d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller */ 125d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller if (start >= limit) { 126d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller start = 0; 127d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller if (iommu->flush_all) 128d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller iommu->flush_all(iommu); 129d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller } 130d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller 131d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller again: 132d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller 133d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller if (dev) 134d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, 135d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller 1 << IO_PAGE_SHIFT); 136d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller else 137d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller boundary_size = ALIGN(1UL << 32, 1 << IO_PAGE_SHIFT); 138d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller 13989c94f2f70d093f59b55d3ea8042d13889169346FUJITA Tomonori n = iommu_area_alloc(arena->map, limit, start, npages, 14089c94f2f70d093f59b55d3ea8042d13889169346FUJITA Tomonori iommu->page_table_map_base >> IO_PAGE_SHIFT, 141d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller boundary_size >> IO_PAGE_SHIFT, 0); 142d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller if (n == -1) { 143688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller if (likely(pass < 1)) { 144d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller /* First failure, rescan from the beginning. */ 145688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller start = 0; 146d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller if (iommu->flush_all) 147d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller iommu->flush_all(iommu); 148688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller pass++; 149688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller goto again; 150688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller } else { 151d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller /* Second failure, give up */ 152d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller return DMA_ERROR_CODE; 153688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller } 154688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller } 155688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller 156d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller end = n + npages; 157688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller 158688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller arena->hint = end; 159688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller 160d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller /* Update handle for SG allocations */ 161d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller if (handle) 162d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller *handle = end; 163d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller 164688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller return n; 165688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller} 166688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller 167d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Millervoid iommu_range_free(struct iommu *iommu, dma_addr_t dma_addr, unsigned long npages) 168688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller{ 169d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller struct iommu_arena *arena = &iommu->arena; 170d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller unsigned long entry; 171688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller 172d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller entry = (dma_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT; 173d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller 174a66022c457755b5eef61e30866114679c95e1f54Akinobu Mita bitmap_clear(arena->map, entry, npages); 175688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller} 176688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller 177ad7ad57c6127042c411353dddb723765964815dbDavid S. Millerint iommu_table_init(struct iommu *iommu, int tsbsize, 178c1b1a5f1f1b2612b69b67381b223bce9f8ec4da5David S. Miller u32 dma_offset, u32 dma_addr_mask, 179c1b1a5f1f1b2612b69b67381b223bce9f8ec4da5David S. Miller int numa_node) 1801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 181c1b1a5f1f1b2612b69b67381b223bce9f8ec4da5David S. Miller unsigned long i, order, sz, num_tsb_entries; 182c1b1a5f1f1b2612b69b67381b223bce9f8ec4da5David S. Miller struct page *page; 183688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller 184688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller num_tsb_entries = tsbsize / sizeof(iopte_t); 18551e8513615ed8202b22ba9a43b0c7376ea4f6868David S. Miller 18651e8513615ed8202b22ba9a43b0c7376ea4f6868David S. Miller /* Setup initial software IOMMU state. */ 18751e8513615ed8202b22ba9a43b0c7376ea4f6868David S. Miller spin_lock_init(&iommu->lock); 18851e8513615ed8202b22ba9a43b0c7376ea4f6868David S. Miller iommu->ctx_lowest_free = 1; 18951e8513615ed8202b22ba9a43b0c7376ea4f6868David S. Miller iommu->page_table_map_base = dma_offset; 19051e8513615ed8202b22ba9a43b0c7376ea4f6868David S. Miller iommu->dma_addr_mask = dma_addr_mask; 19151e8513615ed8202b22ba9a43b0c7376ea4f6868David S. Miller 192688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller /* Allocate and initialize the free area map. */ 193688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller sz = num_tsb_entries / 8; 194688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller sz = (sz + 7UL) & ~7UL; 195c1b1a5f1f1b2612b69b67381b223bce9f8ec4da5David S. Miller iommu->arena.map = kmalloc_node(sz, GFP_KERNEL, numa_node); 196688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller if (!iommu->arena.map) { 197ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller printk(KERN_ERR "IOMMU: Error, kmalloc(arena.map) failed.\n"); 198ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller return -ENOMEM; 19951e8513615ed8202b22ba9a43b0c7376ea4f6868David S. Miller } 200c1b1a5f1f1b2612b69b67381b223bce9f8ec4da5David S. Miller memset(iommu->arena.map, 0, sz); 201688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller iommu->arena.limit = num_tsb_entries; 2021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 203d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller if (tlb_type != hypervisor) 204d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller iommu->flush_all = iommu_flushall; 205d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller 20651e8513615ed8202b22ba9a43b0c7376ea4f6868David S. Miller /* Allocate and initialize the dummy page which we 20751e8513615ed8202b22ba9a43b0c7376ea4f6868David S. Miller * set inactive IO PTEs to point to. 20851e8513615ed8202b22ba9a43b0c7376ea4f6868David S. Miller */ 209c1b1a5f1f1b2612b69b67381b223bce9f8ec4da5David S. Miller page = alloc_pages_node(numa_node, GFP_KERNEL, 0); 210c1b1a5f1f1b2612b69b67381b223bce9f8ec4da5David S. Miller if (!page) { 211ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller printk(KERN_ERR "IOMMU: Error, gfp(dummy_page) failed.\n"); 212ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller goto out_free_map; 21351e8513615ed8202b22ba9a43b0c7376ea4f6868David S. Miller } 214c1b1a5f1f1b2612b69b67381b223bce9f8ec4da5David S. Miller iommu->dummy_page = (unsigned long) page_address(page); 215c1b1a5f1f1b2612b69b67381b223bce9f8ec4da5David S. Miller memset((void *)iommu->dummy_page, 0, PAGE_SIZE); 21651e8513615ed8202b22ba9a43b0c7376ea4f6868David S. Miller iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page); 21751e8513615ed8202b22ba9a43b0c7376ea4f6868David S. Miller 21851e8513615ed8202b22ba9a43b0c7376ea4f6868David S. Miller /* Now allocate and setup the IOMMU page table itself. */ 21951e8513615ed8202b22ba9a43b0c7376ea4f6868David S. Miller order = get_order(tsbsize); 220c1b1a5f1f1b2612b69b67381b223bce9f8ec4da5David S. Miller page = alloc_pages_node(numa_node, GFP_KERNEL, order); 221c1b1a5f1f1b2612b69b67381b223bce9f8ec4da5David S. Miller if (!page) { 222ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller printk(KERN_ERR "IOMMU: Error, gfp(tsb) failed.\n"); 223ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller goto out_free_dummy_page; 22451e8513615ed8202b22ba9a43b0c7376ea4f6868David S. Miller } 225c1b1a5f1f1b2612b69b67381b223bce9f8ec4da5David S. Miller iommu->page_table = (iopte_t *)page_address(page); 2261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 227688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller for (i = 0; i < num_tsb_entries; i++) 2281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds iopte_make_dummy(iommu, &iommu->page_table[i]); 229ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller 230ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller return 0; 231ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller 232ad7ad57c6127042c411353dddb723765964815dbDavid S. Millerout_free_dummy_page: 233ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller free_page(iommu->dummy_page); 234ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller iommu->dummy_page = 0UL; 235ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller 236ad7ad57c6127042c411353dddb723765964815dbDavid S. Millerout_free_map: 237ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller kfree(iommu->arena.map); 238ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller iommu->arena.map = NULL; 239ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller 240ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller return -ENOMEM; 2411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 243d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Millerstatic inline iopte_t *alloc_npages(struct device *dev, struct iommu *iommu, 244d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller unsigned long npages) 2451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 246d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller unsigned long entry; 2471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 248d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller entry = iommu_range_alloc(dev, iommu, npages, NULL); 249d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller if (unlikely(entry == DMA_ERROR_CODE)) 250688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller return NULL; 2511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 252688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller return iommu->page_table + entry; 2531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 25516ce82d846f2e6b652a064f91c5019cfe8682be4David S. Millerstatic int iommu_alloc_ctx(struct iommu *iommu) 2567c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller{ 2577c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller int lowest = iommu->ctx_lowest_free; 258711c71a092ccedf5e24cff25e577bfa0148fce66Akinobu Mita int n = find_next_zero_bit(iommu->ctx_bitmap, IOMMU_NUM_CTXS, lowest); 2597c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller 260711c71a092ccedf5e24cff25e577bfa0148fce66Akinobu Mita if (unlikely(n == IOMMU_NUM_CTXS)) { 2617c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller n = find_next_zero_bit(iommu->ctx_bitmap, lowest, 1); 2627c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller if (unlikely(n == lowest)) { 2637c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller printk(KERN_WARNING "IOMMU: Ran out of contexts.\n"); 2647c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller n = 0; 2657c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller } 2667c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller } 2677c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller if (n) 2687c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller __set_bit(n, iommu->ctx_bitmap); 2697c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller 2707c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller return n; 2717c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller} 2727c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller 27316ce82d846f2e6b652a064f91c5019cfe8682be4David S. Millerstatic inline void iommu_free_ctx(struct iommu *iommu, int ctx) 2747c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller{ 2757c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller if (likely(ctx)) { 2767c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller __clear_bit(ctx, iommu->ctx_bitmap); 2777c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller if (ctx < iommu->ctx_lowest_free) 2787c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller iommu->ctx_lowest_free = ctx; 2797c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller } 2807c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller} 2817c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller 282ad7ad57c6127042c411353dddb723765964815dbDavid S. Millerstatic void *dma_4u_alloc_coherent(struct device *dev, size_t size, 283c416258a6e1e68a33fd328e872007d19941138c5Andrzej Pietrasiewicz dma_addr_t *dma_addrp, gfp_t gfp, 284c416258a6e1e68a33fd328e872007d19941138c5Andrzej Pietrasiewicz struct dma_attrs *attrs) 2851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 286c1b1a5f1f1b2612b69b67381b223bce9f8ec4da5David S. Miller unsigned long flags, order, first_page; 28716ce82d846f2e6b652a064f91c5019cfe8682be4David S. Miller struct iommu *iommu; 288c1b1a5f1f1b2612b69b67381b223bce9f8ec4da5David S. Miller struct page *page; 289c1b1a5f1f1b2612b69b67381b223bce9f8ec4da5David S. Miller int npages, nid; 2901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds iopte_t *iopte; 2911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *ret; 2921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds size = IO_PAGE_ALIGN(size); 2941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds order = get_order(size); 2951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (order >= 10) 2961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 2971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 298c1b1a5f1f1b2612b69b67381b223bce9f8ec4da5David S. Miller nid = dev->archdata.numa_node; 299c1b1a5f1f1b2612b69b67381b223bce9f8ec4da5David S. Miller page = alloc_pages_node(nid, gfp, order); 300c1b1a5f1f1b2612b69b67381b223bce9f8ec4da5David S. Miller if (unlikely(!page)) 3011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 302c1b1a5f1f1b2612b69b67381b223bce9f8ec4da5David S. Miller 303c1b1a5f1f1b2612b69b67381b223bce9f8ec4da5David S. Miller first_page = (unsigned long) page_address(page); 3041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds memset((char *)first_page, 0, PAGE_SIZE << order); 3051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 306ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller iommu = dev->archdata.iommu; 3071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&iommu->lock, flags); 309d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller iopte = alloc_npages(dev, iommu, size >> IO_PAGE_SHIFT); 310688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller spin_unlock_irqrestore(&iommu->lock, flags); 311688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller 312688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller if (unlikely(iopte == NULL)) { 3131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds free_pages(first_page, order); 3141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 3151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *dma_addrp = (iommu->page_table_map_base + 3181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ((iopte - iommu->page_table) << IO_PAGE_SHIFT)); 3191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ret = (void *) first_page; 3201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds npages = size >> IO_PAGE_SHIFT; 3211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds first_page = __pa(first_page); 3221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (npages--) { 323688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller iopte_val(*iopte) = (IOPTE_CONSISTENT(0UL) | 3241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds IOPTE_WRITE | 3251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds (first_page & IOPTE_PAGE)); 3261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds iopte++; 3271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds first_page += IO_PAGE_SIZE; 3281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return ret; 3311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 333ad7ad57c6127042c411353dddb723765964815dbDavid S. Millerstatic void dma_4u_free_coherent(struct device *dev, size_t size, 334c416258a6e1e68a33fd328e872007d19941138c5Andrzej Pietrasiewicz void *cpu, dma_addr_t dvma, 335c416258a6e1e68a33fd328e872007d19941138c5Andrzej Pietrasiewicz struct dma_attrs *attrs) 3361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 33716ce82d846f2e6b652a064f91c5019cfe8682be4David S. Miller struct iommu *iommu; 338688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller unsigned long flags, order, npages; 3391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; 341ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller iommu = dev->archdata.iommu; 3421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&iommu->lock, flags); 3441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 345d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller iommu_range_free(iommu, dvma, npages); 3467c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller 3471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&iommu->lock, flags); 3481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds order = get_order(size); 3501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (order < 10) 3511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds free_pages((unsigned long)cpu, order); 3521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 354797a75686528e9f6f9bfee2a719a00b47868c999FUJITA Tomonoristatic dma_addr_t dma_4u_map_page(struct device *dev, struct page *page, 355797a75686528e9f6f9bfee2a719a00b47868c999FUJITA Tomonori unsigned long offset, size_t sz, 356bc0a14f154069cc4e42ea903c2c2b9984a94e4b7FUJITA Tomonori enum dma_data_direction direction, 357bc0a14f154069cc4e42ea903c2c2b9984a94e4b7FUJITA Tomonori struct dma_attrs *attrs) 3581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 35916ce82d846f2e6b652a064f91c5019cfe8682be4David S. Miller struct iommu *iommu; 36016ce82d846f2e6b652a064f91c5019cfe8682be4David S. Miller struct strbuf *strbuf; 3611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds iopte_t *base; 3621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags, npages, oaddr; 3631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long i, base_paddr, ctx; 3641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds u32 bus_addr, ret; 3651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long iopte_protection; 3661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 367ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller iommu = dev->archdata.iommu; 368ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller strbuf = dev->archdata.stc; 3691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 370ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller if (unlikely(direction == DMA_NONE)) 371688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller goto bad_no_ctx; 3721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 373797a75686528e9f6f9bfee2a719a00b47868c999FUJITA Tomonori oaddr = (unsigned long)(page_address(page) + offset); 3741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); 3751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds npages >>= IO_PAGE_SHIFT; 3761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&iommu->lock, flags); 378d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller base = alloc_npages(dev, iommu, npages); 379688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller ctx = 0; 380688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller if (iommu->iommu_ctxflush) 381688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller ctx = iommu_alloc_ctx(iommu); 382688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller spin_unlock_irqrestore(&iommu->lock, flags); 3831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 384688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller if (unlikely(!base)) 3851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto bad; 386688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller 3871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bus_addr = (iommu->page_table_map_base + 3881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ((base - iommu->page_table) << IO_PAGE_SHIFT)); 3891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ret = bus_addr | (oaddr & ~IO_PAGE_MASK); 3901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds base_paddr = __pa(oaddr & IO_PAGE_MASK); 3911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (strbuf->strbuf_enabled) 3921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds iopte_protection = IOPTE_STREAMING(ctx); 3931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds else 3941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds iopte_protection = IOPTE_CONSISTENT(ctx); 395ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller if (direction != DMA_TO_DEVICE) 3961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds iopte_protection |= IOPTE_WRITE; 3971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds for (i = 0; i < npages; i++, base++, base_paddr += IO_PAGE_SIZE) 3991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds iopte_val(*base) = iopte_protection | base_paddr; 4001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return ret; 4021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsbad: 404688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller iommu_free_ctx(iommu, ctx); 405688cb30bdc3e398d97682a6a58f825821ee838c2David S. Millerbad_no_ctx: 406688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller if (printk_ratelimit()) 407688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller WARN_ON(1); 408ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller return DMA_ERROR_CODE; 4091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 411ad7ad57c6127042c411353dddb723765964815dbDavid S. Millerstatic void strbuf_flush(struct strbuf *strbuf, struct iommu *iommu, 412ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller u32 vaddr, unsigned long ctx, unsigned long npages, 413ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller enum dma_data_direction direction) 4144dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller{ 4154dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller int limit; 4164dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller 4174dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller if (strbuf->strbuf_ctxflush && 4184dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller iommu->iommu_ctxflush) { 4194dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller unsigned long matchreg, flushreg; 4207c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller u64 val; 4214dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller 4224dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller flushreg = strbuf->strbuf_ctxflush; 423ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller matchreg = STC_CTXMATCH_ADDR(strbuf, ctx); 4244dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller 425ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller iommu_write(flushreg, ctx); 426ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller val = iommu_read(matchreg); 42788314ee73fd75eb32abdcb3119cd303c116d4500David S. Miller val &= 0xffff; 42888314ee73fd75eb32abdcb3119cd303c116d4500David S. Miller if (!val) 4297c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller goto do_flush_sync; 4307c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller 4317c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller while (val) { 4327c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller if (val & 0x1) 433ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller iommu_write(flushreg, ctx); 4347c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller val >>= 1; 435a228dfd5dc4b92288ea22d427b2bfc48ba5bb8b0David S. Miller } 436ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller val = iommu_read(matchreg); 4377c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller if (unlikely(val)) { 438ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller printk(KERN_WARNING "strbuf_flush: ctx flush " 4399018113649348c689da107166c05d436cd52e7bfSam Ravnborg "timeout matchreg[%llx] ctx[%lx]\n", 4407c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller val, ctx); 4417c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller goto do_page_flush; 4427c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller } 4434dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller } else { 4444dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller unsigned long i; 4454dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller 4467c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller do_page_flush: 4474dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller for (i = 0; i < npages; i++, vaddr += IO_PAGE_SIZE) 448ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller iommu_write(strbuf->strbuf_pflush, vaddr); 4494dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller } 4504dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller 4517c963ad1d113790a8c723a178988b675868f3abeDavid S. Millerdo_flush_sync: 4527c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller /* If the device could not have possibly put dirty data into 4537c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller * the streaming cache, no flush-flag synchronization needs 4547c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller * to be performed. 4557c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller */ 456ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller if (direction == DMA_TO_DEVICE) 4577c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller return; 4587c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller 459ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller STC_FLUSHFLAG_INIT(strbuf); 460ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller iommu_write(strbuf->strbuf_fsync, strbuf->strbuf_flushflag_pa); 461ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller (void) iommu_read(iommu->write_complete_reg); 4624dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller 463a228dfd5dc4b92288ea22d427b2bfc48ba5bb8b0David S. Miller limit = 100000; 464ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller while (!STC_FLUSHFLAG_SET(strbuf)) { 4654dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller limit--; 4664dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller if (!limit) 4674dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller break; 468a228dfd5dc4b92288ea22d427b2bfc48ba5bb8b0David S. Miller udelay(1); 4694f07118f656c179740cad35b827032e2e29b1210David S. Miller rmb(); 4704dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller } 4714dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller if (!limit) 472ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller printk(KERN_WARNING "strbuf_flush: flushflag timeout " 4734dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller "vaddr[%08x] ctx[%lx] npages[%ld]\n", 4744dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller vaddr, ctx, npages); 4754dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller} 4764dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller 477797a75686528e9f6f9bfee2a719a00b47868c999FUJITA Tomonoristatic void dma_4u_unmap_page(struct device *dev, dma_addr_t bus_addr, 478bc0a14f154069cc4e42ea903c2c2b9984a94e4b7FUJITA Tomonori size_t sz, enum dma_data_direction direction, 479bc0a14f154069cc4e42ea903c2c2b9984a94e4b7FUJITA Tomonori struct dma_attrs *attrs) 4801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 48116ce82d846f2e6b652a064f91c5019cfe8682be4David S. Miller struct iommu *iommu; 48216ce82d846f2e6b652a064f91c5019cfe8682be4David S. Miller struct strbuf *strbuf; 4831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds iopte_t *base; 484688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller unsigned long flags, npages, ctx, i; 4851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 486ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller if (unlikely(direction == DMA_NONE)) { 487688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller if (printk_ratelimit()) 488688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller WARN_ON(1); 489688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller return; 490688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller } 4911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 492ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller iommu = dev->archdata.iommu; 493ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller strbuf = dev->archdata.stc; 4941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); 4961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds npages >>= IO_PAGE_SHIFT; 4971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds base = iommu->page_table + 4981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); 4991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bus_addr &= IO_PAGE_MASK; 5001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&iommu->lock, flags); 5021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Record the context, if any. */ 5041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ctx = 0; 5051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (iommu->iommu_ctxflush) 5061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL; 5071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Step 1: Kick data out of streaming buffers if necessary. */ 5094dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller if (strbuf->strbuf_enabled) 510ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller strbuf_flush(strbuf, iommu, bus_addr, ctx, 511ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller npages, direction); 5121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 513688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller /* Step 2: Clear out TSB entries. */ 514688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller for (i = 0; i < npages; i++) 515688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller iopte_make_dummy(iommu, base + i); 5161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 517d284142cbad66832d5072a0aebeca7bd9ca841b7David S. Miller iommu_range_free(iommu, bus_addr, npages); 5181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5197c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller iommu_free_ctx(iommu, ctx); 5207c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller 5211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&iommu->lock, flags); 5221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 5231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 524ad7ad57c6127042c411353dddb723765964815dbDavid S. Millerstatic int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, 525bc0a14f154069cc4e42ea903c2c2b9984a94e4b7FUJITA Tomonori int nelems, enum dma_data_direction direction, 526bc0a14f154069cc4e42ea903c2c2b9984a94e4b7FUJITA Tomonori struct dma_attrs *attrs) 5271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 52813fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller struct scatterlist *s, *outs, *segstart; 52913fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller unsigned long flags, handle, prot, ctx; 53013fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller dma_addr_t dma_next = 0, dma_addr; 53113fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller unsigned int max_seg_size; 532f08802572965873af97e74337d5740bfa2542941FUJITA Tomonori unsigned long seg_boundary_size; 53313fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller int outcount, incount, i; 53416ce82d846f2e6b652a064f91c5019cfe8682be4David S. Miller struct strbuf *strbuf; 53538192d52f159bc06b7f523800c10b583cdd661d5David S. Miller struct iommu *iommu; 536f08802572965873af97e74337d5740bfa2542941FUJITA Tomonori unsigned long base_shift; 53713fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller 53813fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller BUG_ON(direction == DMA_NONE); 5391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 540ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller iommu = dev->archdata.iommu; 541ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller strbuf = dev->archdata.stc; 54213fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller if (nelems == 0 || !iommu) 54313fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller return 0; 5441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&iommu->lock, flags); 5461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 547688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller ctx = 0; 548688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller if (iommu->iommu_ctxflush) 549688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller ctx = iommu_alloc_ctx(iommu); 550688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller 5511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (strbuf->strbuf_enabled) 55213fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller prot = IOPTE_STREAMING(ctx); 5531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds else 55413fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller prot = IOPTE_CONSISTENT(ctx); 555ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller if (direction != DMA_TO_DEVICE) 55613fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller prot |= IOPTE_WRITE; 55713fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller 55813fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller outs = s = segstart = &sglist[0]; 55913fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller outcount = 1; 56013fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller incount = nelems; 56113fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller handle = 0; 56213fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller 56313fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller /* Init first segment length for backout at failure */ 56413fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller outs->dma_length = 0; 56513fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller 56613fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller max_seg_size = dma_get_max_seg_size(dev); 567f08802572965873af97e74337d5740bfa2542941FUJITA Tomonori seg_boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, 568f08802572965873af97e74337d5740bfa2542941FUJITA Tomonori IO_PAGE_SIZE) >> IO_PAGE_SHIFT; 569f08802572965873af97e74337d5740bfa2542941FUJITA Tomonori base_shift = iommu->page_table_map_base >> IO_PAGE_SHIFT; 57013fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller for_each_sg(sglist, s, nelems, i) { 571f08802572965873af97e74337d5740bfa2542941FUJITA Tomonori unsigned long paddr, npages, entry, out_entry = 0, slen; 57213fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller iopte_t *base; 57313fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller 57413fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller slen = s->length; 57513fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller /* Sanity check */ 57613fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller if (slen == 0) { 57713fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller dma_next = 0; 57813fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller continue; 57913fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller } 58013fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller /* Allocate iommu entries for that segment */ 58113fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s); 5820fcff28f47194445f37264d750dbb13d3d894d0bJoerg Roedel npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE); 58313fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller entry = iommu_range_alloc(dev, iommu, npages, &handle); 58413fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller 58513fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller /* Handle failure */ 58613fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller if (unlikely(entry == DMA_ERROR_CODE)) { 58713fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller if (printk_ratelimit()) 58813fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller printk(KERN_INFO "iommu_alloc failed, iommu %p paddr %lx" 58913fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller " npages %lx\n", iommu, paddr, npages); 59013fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller goto iommu_map_failed; 59113fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller } 592688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller 59313fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller base = iommu->page_table + entry; 5941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 59513fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller /* Convert entry to a dma_addr_t */ 59613fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller dma_addr = iommu->page_table_map_base + 59713fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller (entry << IO_PAGE_SHIFT); 59813fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller dma_addr |= (s->offset & ~IO_PAGE_MASK); 59938192d52f159bc06b7f523800c10b583cdd661d5David S. Miller 60013fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller /* Insert into HW table */ 60138192d52f159bc06b7f523800c10b583cdd661d5David S. Miller paddr &= IO_PAGE_MASK; 60213fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller while (npages--) { 60313fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller iopte_val(*base) = prot | paddr; 60438192d52f159bc06b7f523800c10b583cdd661d5David S. Miller base++; 60538192d52f159bc06b7f523800c10b583cdd661d5David S. Miller paddr += IO_PAGE_SIZE; 60638192d52f159bc06b7f523800c10b583cdd661d5David S. Miller } 60713fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller 60813fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller /* If we are in an open segment, try merging */ 60913fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller if (segstart != s) { 61013fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller /* We cannot merge if: 61113fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller * - allocated dma_addr isn't contiguous to previous allocation 61213fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller */ 61313fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller if ((dma_addr != dma_next) || 614f08802572965873af97e74337d5740bfa2542941FUJITA Tomonori (outs->dma_length + s->length > max_seg_size) || 615f08802572965873af97e74337d5740bfa2542941FUJITA Tomonori (is_span_boundary(out_entry, base_shift, 616f08802572965873af97e74337d5740bfa2542941FUJITA Tomonori seg_boundary_size, outs, s))) { 61713fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller /* Can't merge: create a new segment */ 61813fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller segstart = s; 61913fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller outcount++; 62013fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller outs = sg_next(outs); 62113fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller } else { 62213fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller outs->dma_length += s->length; 62313fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller } 62413fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller } 62513fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller 62613fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller if (segstart == s) { 62713fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller /* This is a new segment, fill entries */ 62813fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller outs->dma_address = dma_addr; 62913fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller outs->dma_length = slen; 630f08802572965873af97e74337d5740bfa2542941FUJITA Tomonori out_entry = entry; 63113fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller } 63213fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller 63313fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller /* Calculate next page pointer for contiguous check */ 63413fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller dma_next = dma_addr + slen; 63538192d52f159bc06b7f523800c10b583cdd661d5David S. Miller } 63638192d52f159bc06b7f523800c10b583cdd661d5David S. Miller 63713fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller spin_unlock_irqrestore(&iommu->lock, flags); 63813fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller 63913fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller if (outcount < incount) { 64013fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller outs = sg_next(outs); 64113fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller outs->dma_address = DMA_ERROR_CODE; 64213fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller outs->dma_length = 0; 64313fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller } 64413fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller 64513fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller return outcount; 64613fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller 64713fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Milleriommu_map_failed: 64813fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller for_each_sg(sglist, s, nelems, i) { 64913fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller if (s->dma_length != 0) { 6506c830fefcc2e9d20f0a6c6aff43c8d333da2ea46David S. Miller unsigned long vaddr, npages, entry, j; 65113fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller iopte_t *base; 65213fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller 65313fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller vaddr = s->dma_address & IO_PAGE_MASK; 6540fcff28f47194445f37264d750dbb13d3d894d0bJoerg Roedel npages = iommu_num_pages(s->dma_address, s->dma_length, 6550fcff28f47194445f37264d750dbb13d3d894d0bJoerg Roedel IO_PAGE_SIZE); 65613fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller iommu_range_free(iommu, vaddr, npages); 65713fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller 65813fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller entry = (vaddr - iommu->page_table_map_base) 65913fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller >> IO_PAGE_SHIFT; 66013fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller base = iommu->page_table + entry; 66113fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller 6626c830fefcc2e9d20f0a6c6aff43c8d333da2ea46David S. Miller for (j = 0; j < npages; j++) 6636c830fefcc2e9d20f0a6c6aff43c8d333da2ea46David S. Miller iopte_make_dummy(iommu, base + j); 66413fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller 66513fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller s->dma_address = DMA_ERROR_CODE; 66613fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller s->dma_length = 0; 66713fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller } 66813fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller if (s == outs) 66913fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller break; 67013fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller } 67113fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller spin_unlock_irqrestore(&iommu->lock, flags); 6721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 673688cb30bdc3e398d97682a6a58f825821ee838c2David S. Miller return 0; 6741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 6751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 67613fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller/* If contexts are being used, they are the same in all of the mappings 67713fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller * we make for a particular SG. 67813fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller */ 67913fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Millerstatic unsigned long fetch_sg_ctx(struct iommu *iommu, struct scatterlist *sg) 68013fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller{ 68113fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller unsigned long ctx = 0; 68213fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller 68313fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller if (iommu->iommu_ctxflush) { 68413fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller iopte_t *base; 68513fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller u32 bus_addr; 68613fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller 68713fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller bus_addr = sg->dma_address & IO_PAGE_MASK; 68813fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller base = iommu->page_table + 68913fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); 69013fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller 69113fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL; 69213fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller } 69313fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller return ctx; 69413fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller} 69513fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller 696ad7ad57c6127042c411353dddb723765964815dbDavid S. Millerstatic void dma_4u_unmap_sg(struct device *dev, struct scatterlist *sglist, 697bc0a14f154069cc4e42ea903c2c2b9984a94e4b7FUJITA Tomonori int nelems, enum dma_data_direction direction, 698bc0a14f154069cc4e42ea903c2c2b9984a94e4b7FUJITA Tomonori struct dma_attrs *attrs) 6991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 70013fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller unsigned long flags, ctx; 70113fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller struct scatterlist *sg; 70216ce82d846f2e6b652a064f91c5019cfe8682be4David S. Miller struct strbuf *strbuf; 70338192d52f159bc06b7f523800c10b583cdd661d5David S. Miller struct iommu *iommu; 7041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 70513fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller BUG_ON(direction == DMA_NONE); 7061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 707ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller iommu = dev->archdata.iommu; 708ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller strbuf = dev->archdata.stc; 709ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller 71013fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller ctx = fetch_sg_ctx(iommu, sglist); 7111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 71213fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller spin_lock_irqsave(&iommu->lock, flags); 7131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 71413fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller sg = sglist; 71513fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller while (nelems--) { 71613fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller dma_addr_t dma_handle = sg->dma_address; 71713fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller unsigned int len = sg->dma_length; 71813fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller unsigned long npages, entry; 71913fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller iopte_t *base; 72013fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller int i; 7211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 72213fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller if (!len) 72313fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller break; 7240fcff28f47194445f37264d750dbb13d3d894d0bJoerg Roedel npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE); 72513fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller iommu_range_free(iommu, dma_handle, npages); 7261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 72713fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller entry = ((dma_handle - iommu->page_table_map_base) 72813fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller >> IO_PAGE_SHIFT); 72913fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller base = iommu->page_table + entry; 7301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 73113fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller dma_handle &= IO_PAGE_MASK; 73213fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller if (strbuf->strbuf_enabled) 73313fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller strbuf_flush(strbuf, iommu, dma_handle, ctx, 73413fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller npages, direction); 7351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 73613fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller for (i = 0; i < npages; i++) 73713fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller iopte_make_dummy(iommu, base + i); 7381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 73913fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller sg = sg_next(sg); 74013fa14e185614066d96f90f09da08eebe58cbc8fDavid S. Miller } 7411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7427c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller iommu_free_ctx(iommu, ctx); 7437c963ad1d113790a8c723a178988b675868f3abeDavid S. Miller 7441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&iommu->lock, flags); 7451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 7461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 747ad7ad57c6127042c411353dddb723765964815dbDavid S. Millerstatic void dma_4u_sync_single_for_cpu(struct device *dev, 748ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller dma_addr_t bus_addr, size_t sz, 749ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller enum dma_data_direction direction) 7501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 75116ce82d846f2e6b652a064f91c5019cfe8682be4David S. Miller struct iommu *iommu; 75216ce82d846f2e6b652a064f91c5019cfe8682be4David S. Miller struct strbuf *strbuf; 7531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags, ctx, npages; 7541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 755ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller iommu = dev->archdata.iommu; 756ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller strbuf = dev->archdata.stc; 7571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!strbuf->strbuf_enabled) 7591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return; 7601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&iommu->lock, flags); 7621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); 7641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds npages >>= IO_PAGE_SHIFT; 7651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bus_addr &= IO_PAGE_MASK; 7661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Step 1: Record the context, if any. */ 7681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ctx = 0; 7691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (iommu->iommu_ctxflush && 7701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds strbuf->strbuf_ctxflush) { 7711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds iopte_t *iopte; 7721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds iopte = iommu->page_table + 7741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ((bus_addr - iommu->page_table_map_base)>>IO_PAGE_SHIFT); 7751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL; 7761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 7771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Step 2: Kick data out of streaming buffers. */ 779ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction); 7801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&iommu->lock, flags); 7821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 7831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 784ad7ad57c6127042c411353dddb723765964815dbDavid S. Millerstatic void dma_4u_sync_sg_for_cpu(struct device *dev, 785ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller struct scatterlist *sglist, int nelems, 786ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller enum dma_data_direction direction) 7871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 78816ce82d846f2e6b652a064f91c5019cfe8682be4David S. Miller struct iommu *iommu; 78916ce82d846f2e6b652a064f91c5019cfe8682be4David S. Miller struct strbuf *strbuf; 7904dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller unsigned long flags, ctx, npages, i; 7912c941a204070ab32d92d40318a3196a7fb994c00Jens Axboe struct scatterlist *sg, *sgprv; 7924dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller u32 bus_addr; 7931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 794ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller iommu = dev->archdata.iommu; 795ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller strbuf = dev->archdata.stc; 7961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!strbuf->strbuf_enabled) 7981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return; 7991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&iommu->lock, flags); 8011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Step 1: Record the context, if any. */ 8031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ctx = 0; 8041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (iommu->iommu_ctxflush && 8051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds strbuf->strbuf_ctxflush) { 8061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds iopte_t *iopte; 8071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds iopte = iommu->page_table + 8091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ((sglist[0].dma_address - iommu->page_table_map_base) >> IO_PAGE_SHIFT); 8101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL; 8111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 8121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Step 2: Kick data out of streaming buffers. */ 8144dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller bus_addr = sglist[0].dma_address & IO_PAGE_MASK; 8152c941a204070ab32d92d40318a3196a7fb994c00Jens Axboe sgprv = NULL; 8162c941a204070ab32d92d40318a3196a7fb994c00Jens Axboe for_each_sg(sglist, sg, nelems, i) { 8172c941a204070ab32d92d40318a3196a7fb994c00Jens Axboe if (sg->dma_length == 0) 8184dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller break; 8192c941a204070ab32d92d40318a3196a7fb994c00Jens Axboe sgprv = sg; 8202c941a204070ab32d92d40318a3196a7fb994c00Jens Axboe } 8212c941a204070ab32d92d40318a3196a7fb994c00Jens Axboe 8222c941a204070ab32d92d40318a3196a7fb994c00Jens Axboe npages = (IO_PAGE_ALIGN(sgprv->dma_address + sgprv->dma_length) 8234dbc30fb27ac4e647e6efadb382ff7d38c3d368eDavid S. Miller - bus_addr) >> IO_PAGE_SHIFT; 824ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction); 8251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&iommu->lock, flags); 8271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 8281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 82902f7a18935eef0e56d9180fc3c35da6aad1ff3bbFUJITA Tomonoristatic struct dma_map_ops sun4u_dma_ops = { 830c416258a6e1e68a33fd328e872007d19941138c5Andrzej Pietrasiewicz .alloc = dma_4u_alloc_coherent, 831c416258a6e1e68a33fd328e872007d19941138c5Andrzej Pietrasiewicz .free = dma_4u_free_coherent, 832797a75686528e9f6f9bfee2a719a00b47868c999FUJITA Tomonori .map_page = dma_4u_map_page, 833797a75686528e9f6f9bfee2a719a00b47868c999FUJITA Tomonori .unmap_page = dma_4u_unmap_page, 834ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller .map_sg = dma_4u_map_sg, 835ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller .unmap_sg = dma_4u_unmap_sg, 836ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller .sync_single_for_cpu = dma_4u_sync_single_for_cpu, 837ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller .sync_sg_for_cpu = dma_4u_sync_sg_for_cpu, 8388f6a93a196ba6c569c3e8daa6e81cca7e3ba81b1David S. Miller}; 8398f6a93a196ba6c569c3e8daa6e81cca7e3ba81b1David S. Miller 84002f7a18935eef0e56d9180fc3c35da6aad1ff3bbFUJITA Tomonoristruct dma_map_ops *dma_ops = &sun4u_dma_ops; 841ad7ad57c6127042c411353dddb723765964815dbDavid S. MillerEXPORT_SYMBOL(dma_ops); 8421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 843ee664a9252d24ef10317d1bba8fc8f4c6495b36cFUJITA Tomonoriextern int pci64_dma_supported(struct pci_dev *pdev, u64 device_mask); 844ee664a9252d24ef10317d1bba8fc8f4c6495b36cFUJITA Tomonori 845ad7ad57c6127042c411353dddb723765964815dbDavid S. Millerint dma_supported(struct device *dev, u64 device_mask) 8461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 847ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller struct iommu *iommu = dev->archdata.iommu; 848ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller u64 dma_addr_mask = iommu->dma_addr_mask; 8491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 850ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller if (device_mask >= (1UL << 32UL)) 851ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller return 0; 8521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 853ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller if ((device_mask & dma_addr_mask) == dma_addr_mask) 854ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller return 1; 8551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 856ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller#ifdef CONFIG_PCI 857ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller if (dev->bus == &pci_bus_type) 858ee664a9252d24ef10317d1bba8fc8f4c6495b36cFUJITA Tomonori return pci64_dma_supported(to_pci_dev(dev), device_mask); 859ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller#endif 8601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 861ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller return 0; 862ad7ad57c6127042c411353dddb723765964815dbDavid S. Miller} 863ad7ad57c6127042c411353dddb723765964815dbDavid S. MillerEXPORT_SYMBOL(dma_supported); 864