msm_iommu.c revision 9e28547f887c3040747d8422e1feef79bea93bd8
141f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved. 20720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * 30720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * This program is free software; you can redistribute it and/or modify 40720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * it under the terms of the GNU General Public License version 2 and 50720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * only version 2 as published by the Free Software Foundation. 60720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * 70720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * This program is distributed in the hope that it will be useful, 80720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * but WITHOUT ANY WARRANTY; without even the implied warranty of 90720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * GNU General Public License for more details. 110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * 120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * You should have received a copy of the GNU General Public License 130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * along with this program; if not, write to the Free Software 140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * 02110-1301, USA. 160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko */ 170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/kernel.h> 200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/module.h> 210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/platform_device.h> 220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/errno.h> 230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/io.h> 240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/interrupt.h> 250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/list.h> 260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/spinlock.h> 270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/slab.h> 280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/iommu.h> 2941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko#include <linux/clk.h> 300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <asm/cacheflush.h> 320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <asm/sizes.h> 330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <mach/iommu_hw-8xxx.h> 350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <mach/iommu.h> 360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 37100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko#define MRC(reg, processor, op1, crn, crm, op2) \ 38100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko__asm__ __volatile__ ( \ 39100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko" mrc " #processor "," #op1 ", %0," #crn "," #crm "," #op2 "\n" \ 40100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko: "=r" (reg)) 41100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 42100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko#define RCP15_PRRR(reg) MRC(reg, p15, 0, c10, c2, 0) 43100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko#define RCP15_NMRR(reg) MRC(reg, p15, 0, c10, c2, 1) 44100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 45100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenkostatic int msm_iommu_tex_class[4]; 46100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 470720d1f052dc1576396a39b327da6e60082c4efaStepan MoskovchenkoDEFINE_SPINLOCK(msm_iommu_lock); 480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostruct msm_priv { 500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *pgtable; 510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct list_head list_attached; 520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}; 530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenkostatic int __enable_clocks(struct msm_iommu_drvdata *drvdata) 5541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko{ 5641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko int ret; 5741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 5841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko ret = clk_enable(drvdata->pclk); 5941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (ret) 6041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko goto fail; 6141f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 6241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (drvdata->clk) { 6341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko ret = clk_enable(drvdata->clk); 6441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (ret) 6541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko clk_disable(drvdata->pclk); 6641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko } 6741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenkofail: 6841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko return ret; 6941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko} 7041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 7141f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenkostatic void __disable_clocks(struct msm_iommu_drvdata *drvdata) 7241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko{ 7341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (drvdata->clk) 7441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko clk_disable(drvdata->clk); 7541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko clk_disable(drvdata->pclk); 7641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko} 7741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 7833069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenkostatic int __flush_iotlb(struct iommu_domain *domain) 790720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 800720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv = domain->priv; 810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_drvdata *iommu_drvdata; 820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_drvdata *ctx_drvdata; 8333069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko int ret = 0; 840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#ifndef CONFIG_IOMMU_PGTABLES_L2 850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *fl_table = priv->pgtable; 860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int i; 870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 88f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko if (!list_empty(&priv->list_attached)) { 89f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko dmac_flush_range(fl_table, fl_table + SZ_16K); 900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 91f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko for (i = 0; i < NUM_FL_PTE; i++) 92f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko if ((fl_table[i] & 0x03) == FL_TYPE_TABLE) { 93f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko void *sl_table = __va(fl_table[i] & 94f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko FL_BASE_MASK); 95f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko dmac_flush_range(sl_table, sl_table + SZ_4K); 96f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko } 97f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko } 980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#endif 990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1000720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko list_for_each_entry(ctx_drvdata, &priv->list_attached, attached_elm) { 1010720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!ctx_drvdata->pdev || !ctx_drvdata->pdev->dev.parent) 1020720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko BUG(); 1030720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent); 10541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko BUG_ON(!iommu_drvdata); 10641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 10741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko ret = __enable_clocks(iommu_drvdata); 10841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (ret) 10941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko goto fail; 11041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 1110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_CTX_TLBIALL(iommu_drvdata->base, ctx_drvdata->num, 0); 11241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko __disable_clocks(iommu_drvdata); 1130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 11441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenkofail: 11533069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko return ret; 1160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 1170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void __reset_context(void __iomem *base, int ctx) 1190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 1200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_BPRCOSH(base, ctx, 0); 1210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_BPRCISH(base, ctx, 0); 1220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_BPRCNSH(base, ctx, 0); 1230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_BPSHCFG(base, ctx, 0); 1240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_BPMTCFG(base, ctx, 0); 1250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_ACTLR(base, ctx, 0); 1260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_SCTLR(base, ctx, 0); 1270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_FSRRESTORE(base, ctx, 0); 1280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR0(base, ctx, 0); 1290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR1(base, ctx, 0); 1300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBCR(base, ctx, 0); 1310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_BFBCR(base, ctx, 0); 1320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_PAR(base, ctx, 0); 1330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_FAR(base, ctx, 0); 1340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_CTX_TLBIALL(base, ctx, 0); 1350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TLBFLPTER(base, ctx, 0); 1360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TLBSLPTER(base, ctx, 0); 1370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TLBLKCR(base, ctx, 0); 1380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_PRRR(base, ctx, 0); 1390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_NMRR(base, ctx, 0); 1400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 1410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void __program_context(void __iomem *base, int ctx, phys_addr_t pgtable) 1430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 144100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko unsigned int prrr, nmrr; 1450720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko __reset_context(base, ctx); 1460720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1470720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Set up HTW mode */ 1480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* TLB miss configuration: perform HTW on miss */ 1490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TLBMCFG(base, ctx, 0x3); 1500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* V2P configuration: HTW for access */ 1520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_V2PCFG(base, ctx, 0x3); 1530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBCR(base, ctx, 0); 1550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR0_PA(base, ctx, (pgtable >> 14)); 1560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Invalidate the TLB for this context */ 1580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_CTX_TLBIALL(base, ctx, 0); 1590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Set interrupt number to "secure" interrupt */ 1610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_IRPTNDX(base, ctx, 0); 1620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Enable context fault interrupt */ 1640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_CFEIE(base, ctx, 1); 1650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Stall access on a context fault and let the handler deal with it */ 1670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_CFCFG(base, ctx, 1); 1680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Redirect all cacheable requests to L2 slave port. */ 1700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_RCISH(base, ctx, 1); 1710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_RCOSH(base, ctx, 1); 1720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_RCNSH(base, ctx, 1); 1730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Turn on TEX Remap */ 1750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TRE(base, ctx, 1); 1760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 177100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko /* Set TEX remap attributes */ 178100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko RCP15_PRRR(prrr); 179100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko RCP15_NMRR(nmrr); 180100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko SET_PRRR(base, ctx, prrr); 181100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko SET_NMRR(base, ctx, nmrr); 1820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Turn on BFB prefetch */ 1840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_BFBDFE(base, ctx, 1); 1850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#ifdef CONFIG_IOMMU_PGTABLES_L2 1870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Configure page tables as inner-cacheable and shareable to reduce 1880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * the TLB miss penalty. 1890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko */ 1900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR0_SH(base, ctx, 1); 1910720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR1_SH(base, ctx, 1); 1920720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1930720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR0_NOS(base, ctx, 1); 1940720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR1_NOS(base, ctx, 1); 1950720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR0_IRGNH(base, ctx, 0); /* WB, WA */ 1970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR0_IRGNL(base, ctx, 1); 1980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR1_IRGNH(base, ctx, 0); /* WB, WA */ 2000720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR1_IRGNL(base, ctx, 1); 2010720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2020720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR0_ORGN(base, ctx, 1); /* WB, WA */ 2030720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR1_ORGN(base, ctx, 1); /* WB, WA */ 2040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#endif 2050720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2060720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Enable the MMU */ 2070720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_M(base, ctx, 1); 2080720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 2090720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_domain_init(struct iommu_domain *domain) 2110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 2120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv = kzalloc(sizeof(*priv), GFP_KERNEL); 2130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!priv) 2150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail_nomem; 2160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko INIT_LIST_HEAD(&priv->list_attached); 2180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko priv->pgtable = (unsigned long *)__get_free_pages(GFP_KERNEL, 2190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko get_order(SZ_16K)); 2200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!priv->pgtable) 2220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail_nomem; 2230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko memset(priv->pgtable, 0, SZ_16K); 2250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko domain->priv = priv; 2260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return 0; 2270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail_nomem: 2290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko kfree(priv); 2300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return -ENOMEM; 2310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 2320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void msm_iommu_domain_destroy(struct iommu_domain *domain) 2340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 2350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv; 2360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long flags; 2370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *fl_table; 2380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int i; 2390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_lock_irqsave(&msm_iommu_lock, flags); 2410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko priv = domain->priv; 2420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko domain->priv = NULL; 2430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (priv) { 2450720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fl_table = priv->pgtable; 2460720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2470720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko for (i = 0; i < NUM_FL_PTE; i++) 2480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if ((fl_table[i] & 0x03) == FL_TYPE_TABLE) 2490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko free_page((unsigned long) __va(((fl_table[i]) & 2500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko FL_BASE_MASK))); 2510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko free_pages((unsigned long)priv->pgtable, get_order(SZ_16K)); 2530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko priv->pgtable = NULL; 2540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 2550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko kfree(priv); 2570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_unlock_irqrestore(&msm_iommu_lock, flags); 2580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 2590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) 2610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 2620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv; 2630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_dev *ctx_dev; 2640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_drvdata *iommu_drvdata; 2650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_drvdata *ctx_drvdata; 2660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_drvdata *tmp_drvdata; 2670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int ret = 0; 2680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long flags; 2690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_lock_irqsave(&msm_iommu_lock, flags); 2710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko priv = domain->priv; 2730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!priv || !dev) { 2750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -EINVAL; 2760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 2770720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 2780720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2790720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko iommu_drvdata = dev_get_drvdata(dev->parent); 2800720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ctx_drvdata = dev_get_drvdata(dev); 2810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ctx_dev = dev->platform_data; 2820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!iommu_drvdata || !ctx_drvdata || !ctx_dev) { 2840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -EINVAL; 2850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 2860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 2870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 28800d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko if (!list_empty(&ctx_drvdata->attached_elm)) { 28900d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko ret = -EBUSY; 29000d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko goto fail; 29100d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko } 29200d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko 2930720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko list_for_each_entry(tmp_drvdata, &priv->list_attached, attached_elm) 2940720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (tmp_drvdata == ctx_drvdata) { 2950720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -EBUSY; 2960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 2970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 2980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 29941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko ret = __enable_clocks(iommu_drvdata); 30041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (ret) 30141f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko goto fail; 30241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 3030720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko __program_context(iommu_drvdata->base, ctx_dev->num, 3040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko __pa(priv->pgtable)); 3050720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 30641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko __disable_clocks(iommu_drvdata); 3070720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko list_add(&(ctx_drvdata->attached_elm), &priv->list_attached); 30833069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko ret = __flush_iotlb(domain); 3090720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail: 3110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_unlock_irqrestore(&msm_iommu_lock, flags); 3120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return ret; 3130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 3140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void msm_iommu_detach_dev(struct iommu_domain *domain, 3160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct device *dev) 3170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 3180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv; 3190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_dev *ctx_dev; 3200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_drvdata *iommu_drvdata; 3210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_drvdata *ctx_drvdata; 3220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long flags; 32333069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko int ret; 3240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_lock_irqsave(&msm_iommu_lock, flags); 3260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko priv = domain->priv; 3270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!priv || !dev) 3290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 3300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko iommu_drvdata = dev_get_drvdata(dev->parent); 3320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ctx_drvdata = dev_get_drvdata(dev); 3330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ctx_dev = dev->platform_data; 3340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!iommu_drvdata || !ctx_drvdata || !ctx_dev) 3360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 3370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 33833069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko ret = __flush_iotlb(domain); 33933069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko if (ret) 34033069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko goto fail; 34133069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko 34241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko ret = __enable_clocks(iommu_drvdata); 34341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (ret) 34441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko goto fail; 34541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 3460720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko __reset_context(iommu_drvdata->base, ctx_dev->num); 34741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko __disable_clocks(iommu_drvdata); 3480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko list_del_init(&ctx_drvdata->attached_elm); 3490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail: 3510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_unlock_irqrestore(&msm_iommu_lock, flags); 3520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 3530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_map(struct iommu_domain *domain, unsigned long va, 3550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko phys_addr_t pa, int order, int prot) 3560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 3570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv; 3580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long flags; 3590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *fl_table; 3600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *fl_pte; 3610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long fl_offset; 3620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *sl_table; 3630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *sl_pte; 3640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long sl_offset; 365100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko unsigned int pgprot; 3660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko size_t len = 0x1000UL << order; 367100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko int ret = 0, tex, sh; 3680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_lock_irqsave(&msm_iommu_lock, flags); 3700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 371100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko sh = (prot & MSM_IOMMU_ATTR_SH) ? 1 : 0; 372100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko tex = msm_iommu_tex_class[prot & MSM_IOMMU_CP_MASK]; 373100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 374100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko if (tex < 0 || tex > NUM_TEX_CLASS - 1) { 375100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko ret = -EINVAL; 376100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko goto fail; 377100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko } 378100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 379100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko priv = domain->priv; 3800720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!priv) { 3810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -EINVAL; 3820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 3830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 3840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fl_table = priv->pgtable; 3860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len != SZ_16M && len != SZ_1M && 3880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko len != SZ_64K && len != SZ_4K) { 3890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_debug("Bad size: %d\n", len); 3900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -EINVAL; 3910720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 3920720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 3930720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3940720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!fl_table) { 3950720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_debug("Null page table\n"); 3960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -EINVAL; 3970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 3980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 3990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 400100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko if (len == SZ_16M || len == SZ_1M) { 401100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot = sh ? FL_SHARED : 0; 402100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot |= tex & 0x01 ? FL_BUFFERABLE : 0; 403100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot |= tex & 0x02 ? FL_CACHEABLE : 0; 404100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot |= tex & 0x04 ? FL_TEX0 : 0; 405100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko } else { 406100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot = sh ? SL_SHARED : 0; 407100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot |= tex & 0x01 ? SL_BUFFERABLE : 0; 408100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot |= tex & 0x02 ? SL_CACHEABLE : 0; 409100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot |= tex & 0x04 ? SL_TEX0 : 0; 410100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko } 411100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 4120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fl_offset = FL_OFFSET(va); /* Upper 12 bits */ 4130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */ 4140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_16M) { 4160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int i = 0; 4170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko for (i = 0; i < 16; i++) 4180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *(fl_pte+i) = (pa & 0xFF000000) | FL_SUPERSECTION | 4190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko FL_AP_READ | FL_AP_WRITE | FL_TYPE_SECT | 4202e8c8ba98376459e73d03a285f5d3406b630ea2dStepan Moskovchenko FL_SHARED | FL_NG | pgprot; 4210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 4220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_1M) 4242e8c8ba98376459e73d03a285f5d3406b630ea2dStepan Moskovchenko *fl_pte = (pa & 0xFFF00000) | FL_AP_READ | FL_AP_WRITE | FL_NG | 425100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko FL_TYPE_SECT | FL_SHARED | pgprot; 4260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Need a 2nd level table */ 4280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if ((len == SZ_4K || len == SZ_64K) && (*fl_pte) == 0) { 4290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *sl; 430294b2dea83ba0a6d6034a7521bc62c317efab17bStepan Moskovchenko sl = (unsigned long *) __get_free_pages(GFP_ATOMIC, 4310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko get_order(SZ_4K)); 4320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!sl) { 4340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_debug("Could not allocate second level table\n"); 4350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -ENOMEM; 4360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 4370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 4380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko memset(sl, 0, SZ_4K); 4400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *fl_pte = ((((int)__pa(sl)) & FL_BASE_MASK) | FL_TYPE_TABLE); 4410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 4420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK)); 4440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko sl_offset = SL_OFFSET(va); 4450720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko sl_pte = sl_table + sl_offset; 4460720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4470720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_4K) 4492e8c8ba98376459e73d03a285f5d3406b630ea2dStepan Moskovchenko *sl_pte = (pa & SL_BASE_MASK_SMALL) | SL_AP0 | SL_AP1 | SL_NG | 450100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko SL_SHARED | SL_TYPE_SMALL | pgprot; 4510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_64K) { 4530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int i; 4540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko for (i = 0; i < 16; i++) 4560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *(sl_pte+i) = (pa & SL_BASE_MASK_LARGE) | SL_AP0 | 4572e8c8ba98376459e73d03a285f5d3406b630ea2dStepan Moskovchenko SL_NG | SL_AP1 | SL_SHARED | SL_TYPE_LARGE | pgprot; 4580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 4590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 46033069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko ret = __flush_iotlb(domain); 4610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail: 4620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_unlock_irqrestore(&msm_iommu_lock, flags); 4630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return ret; 4640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 4650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_unmap(struct iommu_domain *domain, unsigned long va, 4670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int order) 4680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 4690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv; 4700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long flags; 4710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *fl_table; 4720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *fl_pte; 4730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long fl_offset; 4740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *sl_table; 4750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *sl_pte; 4760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long sl_offset; 4770720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko size_t len = 0x1000UL << order; 4780720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int i, ret = 0; 4790720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4800720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_lock_irqsave(&msm_iommu_lock, flags); 4810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko priv = domain->priv; 4830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!priv) { 4850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -ENODEV; 4860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 4870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 4880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fl_table = priv->pgtable; 4900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4910720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len != SZ_16M && len != SZ_1M && 4920720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko len != SZ_64K && len != SZ_4K) { 4930720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_debug("Bad length: %d\n", len); 4940720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -EINVAL; 4950720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 4960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 4970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!fl_table) { 4990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_debug("Null page table\n"); 5000720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -EINVAL; 5010720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 5020720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 5030720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fl_offset = FL_OFFSET(va); /* Upper 12 bits */ 5050720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */ 5060720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5070720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (*fl_pte == 0) { 5080720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_debug("First level PTE is 0\n"); 5090720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -ENODEV; 5100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 5110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 5120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Unmap supersection */ 5140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_16M) 5150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko for (i = 0; i < 16; i++) 5160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *(fl_pte+i) = 0; 5170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_1M) 5190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *fl_pte = 0; 5200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK)); 5220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko sl_offset = SL_OFFSET(va); 5230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko sl_pte = sl_table + sl_offset; 5240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_64K) { 5260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko for (i = 0; i < 16; i++) 5270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *(sl_pte+i) = 0; 5280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 5290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_4K) 5310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *sl_pte = 0; 5320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_4K || len == SZ_64K) { 5340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int used = 0; 5350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko for (i = 0; i < NUM_SL_PTE; i++) 5370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (sl_table[i]) 5380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko used = 1; 5390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!used) { 5400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko free_page((unsigned long)sl_table); 5410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *fl_pte = 0; 5420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 5430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 5440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 54533069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko ret = __flush_iotlb(domain); 5469e28547f887c3040747d8422e1feef79bea93bd8Ohad Ben-Cohen 5479e28547f887c3040747d8422e1feef79bea93bd8Ohad Ben-Cohen /* 5489e28547f887c3040747d8422e1feef79bea93bd8Ohad Ben-Cohen * the IOMMU API requires us to return the order of the unmapped 5499e28547f887c3040747d8422e1feef79bea93bd8Ohad Ben-Cohen * page (on success). 5509e28547f887c3040747d8422e1feef79bea93bd8Ohad Ben-Cohen */ 5519e28547f887c3040747d8422e1feef79bea93bd8Ohad Ben-Cohen if (!ret) 5529e28547f887c3040747d8422e1feef79bea93bd8Ohad Ben-Cohen ret = order; 5530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail: 5540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_unlock_irqrestore(&msm_iommu_lock, flags); 5550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return ret; 5560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 5570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain, 5590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long va) 5600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 5610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv; 5620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_drvdata *iommu_drvdata; 5630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_drvdata *ctx_drvdata; 5640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned int par; 5650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long flags; 5660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko void __iomem *base; 5670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko phys_addr_t ret = 0; 5680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int ctx; 5690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_lock_irqsave(&msm_iommu_lock, flags); 5710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko priv = domain->priv; 5730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (list_empty(&priv->list_attached)) 5740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 5750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ctx_drvdata = list_entry(priv->list_attached.next, 5770720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_drvdata, attached_elm); 5780720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent); 5790720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5800720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko base = iommu_drvdata->base; 5810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ctx = ctx_drvdata->num; 5820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 58341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko ret = __enable_clocks(iommu_drvdata); 58441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (ret) 58541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko goto fail; 58641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 5870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Invalidate context TLB */ 5880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_CTX_TLBIALL(base, ctx, 0); 589b0e7808d548ea1d857216d31d63078411203a116Stepan Moskovchenko SET_V2PPR(base, ctx, va & V2Pxx_VA); 5900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5910720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko par = GET_PAR(base, ctx); 5920720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5930720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* We are dealing with a supersection */ 5940720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (GET_NOFAULT_SS(base, ctx)) 5950720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = (par & 0xFF000000) | (va & 0x00FFFFFF); 5960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko else /* Upper 20 bits from PAR, lower 12 from VA */ 5970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = (par & 0xFFFFF000) | (va & 0x00000FFF); 5980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 59933069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko if (GET_FAULT(base, ctx)) 60033069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko ret = 0; 60133069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko 60241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko __disable_clocks(iommu_drvdata); 6030720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail: 6040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_unlock_irqrestore(&msm_iommu_lock, flags); 6050720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return ret; 6060720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 6070720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6080720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_domain_has_cap(struct iommu_domain *domain, 6090720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long cap) 6100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 6110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return 0; 6120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 6130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void print_ctx_regs(void __iomem *base, int ctx) 6150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 6160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned int fsr = GET_FSR(base, ctx); 6170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("FAR = %08x PAR = %08x\n", 6180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko GET_FAR(base, ctx), GET_PAR(base, ctx)); 6190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("FSR = %08x [%s%s%s%s%s%s%s%s%s%s]\n", fsr, 6200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x02) ? "TF " : "", 6210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x04) ? "AFF " : "", 6220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x08) ? "APF " : "", 6230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x10) ? "TLBMF " : "", 6240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x20) ? "HTWDEEF " : "", 6250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x40) ? "HTWSEEF " : "", 6260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x80) ? "MHF " : "", 6270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x10000) ? "SL " : "", 6280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x40000000) ? "SS " : "", 6290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x80000000) ? "MULTI " : ""); 6300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("FSYNR0 = %08x FSYNR1 = %08x\n", 6320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko GET_FSYNR0(base, ctx), GET_FSYNR1(base, ctx)); 6330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("TTBR0 = %08x TTBR1 = %08x\n", 6340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko GET_TTBR0(base, ctx), GET_TTBR1(base, ctx)); 6350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("SCTLR = %08x ACTLR = %08x\n", 6360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko GET_SCTLR(base, ctx), GET_ACTLR(base, ctx)); 6370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("PRRR = %08x NMRR = %08x\n", 6380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko GET_PRRR(base, ctx), GET_NMRR(base, ctx)); 6390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 6400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkoirqreturn_t msm_iommu_fault_handler(int irq, void *dev_id) 6420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 6430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_drvdata *drvdata = dev_id; 6440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko void __iomem *base; 64533069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko unsigned int fsr; 646a43d8c101eb71bf4527dd7f36a34a5a502894f38Stepan Moskovchenko int i, ret; 6470720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_lock(&msm_iommu_lock); 6490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!drvdata) { 6510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("Invalid device ID in context interrupt handler\n"); 6520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 6530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 6540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko base = drvdata->base; 6560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("Unexpected IOMMU page fault!\n"); 6580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("base = %08x\n", (unsigned int) base); 6590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 66041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko ret = __enable_clocks(drvdata); 66141f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (ret) 66241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko goto fail; 66341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 664a43d8c101eb71bf4527dd7f36a34a5a502894f38Stepan Moskovchenko for (i = 0; i < drvdata->ncb; i++) { 6650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fsr = GET_FSR(base, i); 6660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (fsr) { 6670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("Fault occurred in context %d.\n", i); 6680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("Interesting registers:\n"); 6690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko print_ctx_regs(base, i); 6700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_FSR(base, i, 0x4000000F); 6710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 6720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 67341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko __disable_clocks(drvdata); 6740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail: 6750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_unlock(&msm_iommu_lock); 6760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return 0; 6770720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 6780720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6790720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic struct iommu_ops msm_iommu_ops = { 6800720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .domain_init = msm_iommu_domain_init, 6810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .domain_destroy = msm_iommu_domain_destroy, 6820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .attach_dev = msm_iommu_attach_dev, 6830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .detach_dev = msm_iommu_detach_dev, 6840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .map = msm_iommu_map, 6850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .unmap = msm_iommu_unmap, 6860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .iova_to_phys = msm_iommu_iova_to_phys, 6870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .domain_has_cap = msm_iommu_domain_has_cap 6880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}; 6890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 690100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenkostatic int __init get_tex_class(int icp, int ocp, int mt, int nos) 691100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko{ 692100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko int i = 0; 693100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko unsigned int prrr = 0; 694100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko unsigned int nmrr = 0; 695100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko int c_icp, c_ocp, c_mt, c_nos; 696100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 697100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko RCP15_PRRR(prrr); 698100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko RCP15_NMRR(nmrr); 699100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 700100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko for (i = 0; i < NUM_TEX_CLASS; i++) { 701100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko c_nos = PRRR_NOS(prrr, i); 702100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko c_mt = PRRR_MT(prrr, i); 703100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko c_icp = NMRR_ICP(nmrr, i); 704100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko c_ocp = NMRR_OCP(nmrr, i); 705100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 706100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko if (icp == c_icp && ocp == c_ocp && c_mt == mt && c_nos == nos) 707100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko return i; 708100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko } 709100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 710100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko return -ENODEV; 711100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko} 712100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 713100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenkostatic void __init setup_iommu_tex_classes(void) 714100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko{ 715100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko msm_iommu_tex_class[MSM_IOMMU_ATTR_NONCACHED] = 716100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko get_tex_class(CP_NONCACHED, CP_NONCACHED, MT_NORMAL, 1); 717100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 718100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WB_WA] = 719100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko get_tex_class(CP_WB_WA, CP_WB_WA, MT_NORMAL, 1); 720100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 721100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WB_NWA] = 722100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko get_tex_class(CP_WB_NWA, CP_WB_NWA, MT_NORMAL, 1); 723100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 724100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WT] = 725100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko get_tex_class(CP_WT, CP_WT, MT_NORMAL, 1); 726100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko} 727100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 728516cbc793eb4be5123289d067b54dfcdabeddb25Stepan Moskovchenkostatic int __init msm_iommu_init(void) 7290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 730100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko setup_iommu_tex_classes(); 7310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko register_iommu(&msm_iommu_ops); 7320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return 0; 7330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 7340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 7350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkosubsys_initcall(msm_iommu_init); 7360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 7370720d1f052dc1576396a39b327da6e60082c4efaStepan MoskovchenkoMODULE_LICENSE("GPL v2"); 7380720d1f052dc1576396a39b327da6e60082c4efaStepan MoskovchenkoMODULE_AUTHOR("Stepan Moskovchenko <stepanm@codeaurora.org>"); 739