msm_iommu.c revision 41f3f5138a5ea71ee603f3d556953fce9aba3074
141f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved. 20720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * 30720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * This program is free software; you can redistribute it and/or modify 40720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * it under the terms of the GNU General Public License version 2 and 50720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * only version 2 as published by the Free Software Foundation. 60720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * 70720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * This program is distributed in the hope that it will be useful, 80720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * but WITHOUT ANY WARRANTY; without even the implied warranty of 90720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * GNU General Public License for more details. 110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * 120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * You should have received a copy of the GNU General Public License 130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * along with this program; if not, write to the Free Software 140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * 02110-1301, USA. 160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko */ 170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/kernel.h> 200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/module.h> 210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/platform_device.h> 220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/errno.h> 230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/io.h> 240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/interrupt.h> 250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/list.h> 260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/spinlock.h> 270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/slab.h> 280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/iommu.h> 2941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko#include <linux/clk.h> 300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <asm/cacheflush.h> 320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <asm/sizes.h> 330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <mach/iommu_hw-8xxx.h> 350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <mach/iommu.h> 360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 37100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko#define MRC(reg, processor, op1, crn, crm, op2) \ 38100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko__asm__ __volatile__ ( \ 39100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko" mrc " #processor "," #op1 ", %0," #crn "," #crm "," #op2 "\n" \ 40100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko: "=r" (reg)) 41100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 42100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko#define RCP15_PRRR(reg) MRC(reg, p15, 0, c10, c2, 0) 43100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko#define RCP15_NMRR(reg) MRC(reg, p15, 0, c10, c2, 1) 44100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 45100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenkostatic int msm_iommu_tex_class[4]; 46100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 470720d1f052dc1576396a39b327da6e60082c4efaStepan MoskovchenkoDEFINE_SPINLOCK(msm_iommu_lock); 480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostruct msm_priv { 500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *pgtable; 510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct list_head list_attached; 520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}; 530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenkostatic int __enable_clocks(struct msm_iommu_drvdata *drvdata) 5541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko{ 5641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko int ret; 5741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 5841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko ret = clk_enable(drvdata->pclk); 5941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (ret) 6041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko goto fail; 6141f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 6241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (drvdata->clk) { 6341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko ret = clk_enable(drvdata->clk); 6441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (ret) 6541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko clk_disable(drvdata->pclk); 6641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko } 6741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenkofail: 6841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko return ret; 6941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko} 7041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 7141f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenkostatic void __disable_clocks(struct msm_iommu_drvdata *drvdata) 7241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko{ 7341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (drvdata->clk) 7441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko clk_disable(drvdata->clk); 7541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko clk_disable(drvdata->pclk); 7641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko} 7741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 7833069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenkostatic int __flush_iotlb(struct iommu_domain *domain) 790720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 800720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv = domain->priv; 810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_drvdata *iommu_drvdata; 820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_drvdata *ctx_drvdata; 8333069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko int ret = 0; 840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#ifndef CONFIG_IOMMU_PGTABLES_L2 850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *fl_table = priv->pgtable; 860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int i; 870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 88f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko if (!list_empty(&priv->list_attached)) { 89f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko dmac_flush_range(fl_table, fl_table + SZ_16K); 900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 91f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko for (i = 0; i < NUM_FL_PTE; i++) 92f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko if ((fl_table[i] & 0x03) == FL_TYPE_TABLE) { 93f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko void *sl_table = __va(fl_table[i] & 94f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko FL_BASE_MASK); 95f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko dmac_flush_range(sl_table, sl_table + SZ_4K); 96f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko } 97f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko } 980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#endif 990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1000720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko list_for_each_entry(ctx_drvdata, &priv->list_attached, attached_elm) { 1010720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!ctx_drvdata->pdev || !ctx_drvdata->pdev->dev.parent) 1020720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko BUG(); 1030720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent); 10541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko BUG_ON(!iommu_drvdata); 10641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 10741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko ret = __enable_clocks(iommu_drvdata); 10841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (ret) 10941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko goto fail; 11041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 1110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_CTX_TLBIALL(iommu_drvdata->base, ctx_drvdata->num, 0); 11241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko __disable_clocks(iommu_drvdata); 1130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 11441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenkofail: 11533069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko return ret; 1160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 1170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void __reset_context(void __iomem *base, int ctx) 1190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 1200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_BPRCOSH(base, ctx, 0); 1210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_BPRCISH(base, ctx, 0); 1220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_BPRCNSH(base, ctx, 0); 1230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_BPSHCFG(base, ctx, 0); 1240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_BPMTCFG(base, ctx, 0); 1250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_ACTLR(base, ctx, 0); 1260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_SCTLR(base, ctx, 0); 1270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_FSRRESTORE(base, ctx, 0); 1280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR0(base, ctx, 0); 1290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR1(base, ctx, 0); 1300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBCR(base, ctx, 0); 1310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_BFBCR(base, ctx, 0); 1320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_PAR(base, ctx, 0); 1330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_FAR(base, ctx, 0); 1340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_CTX_TLBIALL(base, ctx, 0); 1350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TLBFLPTER(base, ctx, 0); 1360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TLBSLPTER(base, ctx, 0); 1370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TLBLKCR(base, ctx, 0); 1380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_PRRR(base, ctx, 0); 1390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_NMRR(base, ctx, 0); 1400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_CONTEXTIDR(base, ctx, 0); 1410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 1420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void __program_context(void __iomem *base, int ctx, phys_addr_t pgtable) 1440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 145100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko unsigned int prrr, nmrr; 1460720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko __reset_context(base, ctx); 1470720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Set up HTW mode */ 1490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* TLB miss configuration: perform HTW on miss */ 1500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TLBMCFG(base, ctx, 0x3); 1510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* V2P configuration: HTW for access */ 1530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_V2PCFG(base, ctx, 0x3); 1540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBCR(base, ctx, 0); 1560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR0_PA(base, ctx, (pgtable >> 14)); 1570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Invalidate the TLB for this context */ 1590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_CTX_TLBIALL(base, ctx, 0); 1600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Set interrupt number to "secure" interrupt */ 1620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_IRPTNDX(base, ctx, 0); 1630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Enable context fault interrupt */ 1650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_CFEIE(base, ctx, 1); 1660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Stall access on a context fault and let the handler deal with it */ 1680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_CFCFG(base, ctx, 1); 1690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Redirect all cacheable requests to L2 slave port. */ 1710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_RCISH(base, ctx, 1); 1720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_RCOSH(base, ctx, 1); 1730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_RCNSH(base, ctx, 1); 1740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Turn on TEX Remap */ 1760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TRE(base, ctx, 1); 1770720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 178100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko /* Set TEX remap attributes */ 179100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko RCP15_PRRR(prrr); 180100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko RCP15_NMRR(nmrr); 181100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko SET_PRRR(base, ctx, prrr); 182100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko SET_NMRR(base, ctx, nmrr); 1830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Turn on BFB prefetch */ 1850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_BFBDFE(base, ctx, 1); 1860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#ifdef CONFIG_IOMMU_PGTABLES_L2 1880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Configure page tables as inner-cacheable and shareable to reduce 1890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * the TLB miss penalty. 1900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko */ 1910720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR0_SH(base, ctx, 1); 1920720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR1_SH(base, ctx, 1); 1930720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1940720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR0_NOS(base, ctx, 1); 1950720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR1_NOS(base, ctx, 1); 1960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR0_IRGNH(base, ctx, 0); /* WB, WA */ 1980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR0_IRGNL(base, ctx, 1); 1990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2000720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR1_IRGNH(base, ctx, 0); /* WB, WA */ 2010720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR1_IRGNL(base, ctx, 1); 2020720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2030720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR0_ORGN(base, ctx, 1); /* WB, WA */ 2040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR1_ORGN(base, ctx, 1); /* WB, WA */ 2050720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#endif 2060720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2070720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Enable the MMU */ 2080720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_M(base, ctx, 1); 2090720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 2100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_domain_init(struct iommu_domain *domain) 2120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 2130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv = kzalloc(sizeof(*priv), GFP_KERNEL); 2140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!priv) 2160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail_nomem; 2170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko INIT_LIST_HEAD(&priv->list_attached); 2190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko priv->pgtable = (unsigned long *)__get_free_pages(GFP_KERNEL, 2200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko get_order(SZ_16K)); 2210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!priv->pgtable) 2230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail_nomem; 2240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko memset(priv->pgtable, 0, SZ_16K); 2260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko domain->priv = priv; 2270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return 0; 2280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail_nomem: 2300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko kfree(priv); 2310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return -ENOMEM; 2320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 2330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void msm_iommu_domain_destroy(struct iommu_domain *domain) 2350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 2360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv; 2370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long flags; 2380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *fl_table; 2390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int i; 2400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_lock_irqsave(&msm_iommu_lock, flags); 2420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko priv = domain->priv; 2430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko domain->priv = NULL; 2440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2450720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (priv) { 2460720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fl_table = priv->pgtable; 2470720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko for (i = 0; i < NUM_FL_PTE; i++) 2490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if ((fl_table[i] & 0x03) == FL_TYPE_TABLE) 2500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko free_page((unsigned long) __va(((fl_table[i]) & 2510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko FL_BASE_MASK))); 2520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko free_pages((unsigned long)priv->pgtable, get_order(SZ_16K)); 2540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko priv->pgtable = NULL; 2550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 2560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko kfree(priv); 2580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_unlock_irqrestore(&msm_iommu_lock, flags); 2590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 2600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) 2620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 2630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv; 2640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_dev *ctx_dev; 2650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_drvdata *iommu_drvdata; 2660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_drvdata *ctx_drvdata; 2670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_drvdata *tmp_drvdata; 2680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int ret = 0; 2690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long flags; 2700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_lock_irqsave(&msm_iommu_lock, flags); 2720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko priv = domain->priv; 2740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!priv || !dev) { 2760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -EINVAL; 2770720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 2780720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 2790720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2800720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko iommu_drvdata = dev_get_drvdata(dev->parent); 2810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ctx_drvdata = dev_get_drvdata(dev); 2820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ctx_dev = dev->platform_data; 2830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!iommu_drvdata || !ctx_drvdata || !ctx_dev) { 2850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -EINVAL; 2860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 2870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 2880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 28900d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko if (!list_empty(&ctx_drvdata->attached_elm)) { 29000d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko ret = -EBUSY; 29100d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko goto fail; 29200d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko } 29300d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko 2940720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko list_for_each_entry(tmp_drvdata, &priv->list_attached, attached_elm) 2950720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (tmp_drvdata == ctx_drvdata) { 2960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -EBUSY; 2970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 2980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 2990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 30041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko ret = __enable_clocks(iommu_drvdata); 30141f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (ret) 30241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko goto fail; 30341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 3040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko __program_context(iommu_drvdata->base, ctx_dev->num, 3050720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko __pa(priv->pgtable)); 3060720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 30741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko __disable_clocks(iommu_drvdata); 3080720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko list_add(&(ctx_drvdata->attached_elm), &priv->list_attached); 30933069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko ret = __flush_iotlb(domain); 3100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail: 3120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_unlock_irqrestore(&msm_iommu_lock, flags); 3130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return ret; 3140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 3150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void msm_iommu_detach_dev(struct iommu_domain *domain, 3170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct device *dev) 3180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 3190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv; 3200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_dev *ctx_dev; 3210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_drvdata *iommu_drvdata; 3220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_drvdata *ctx_drvdata; 3230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long flags; 32433069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko int ret; 3250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_lock_irqsave(&msm_iommu_lock, flags); 3270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko priv = domain->priv; 3280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!priv || !dev) 3300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 3310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko iommu_drvdata = dev_get_drvdata(dev->parent); 3330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ctx_drvdata = dev_get_drvdata(dev); 3340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ctx_dev = dev->platform_data; 3350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!iommu_drvdata || !ctx_drvdata || !ctx_dev) 3370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 3380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 33933069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko ret = __flush_iotlb(domain); 34033069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko if (ret) 34133069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko goto fail; 34233069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko 34341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko ret = __enable_clocks(iommu_drvdata); 34441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (ret) 34541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko goto fail; 34641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 3470720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko __reset_context(iommu_drvdata->base, ctx_dev->num); 34841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko __disable_clocks(iommu_drvdata); 3490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko list_del_init(&ctx_drvdata->attached_elm); 3500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail: 3520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_unlock_irqrestore(&msm_iommu_lock, flags); 3530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 3540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_map(struct iommu_domain *domain, unsigned long va, 3560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko phys_addr_t pa, int order, int prot) 3570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 3580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv; 3590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long flags; 3600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *fl_table; 3610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *fl_pte; 3620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long fl_offset; 3630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *sl_table; 3640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *sl_pte; 3650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long sl_offset; 366100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko unsigned int pgprot; 3670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko size_t len = 0x1000UL << order; 368100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko int ret = 0, tex, sh; 3690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_lock_irqsave(&msm_iommu_lock, flags); 3710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 372100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko sh = (prot & MSM_IOMMU_ATTR_SH) ? 1 : 0; 373100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko tex = msm_iommu_tex_class[prot & MSM_IOMMU_CP_MASK]; 374100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 375100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko if (tex < 0 || tex > NUM_TEX_CLASS - 1) { 376100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko ret = -EINVAL; 377100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko goto fail; 378100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko } 379100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 380100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko priv = domain->priv; 3810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!priv) { 3820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -EINVAL; 3830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 3840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 3850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fl_table = priv->pgtable; 3870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len != SZ_16M && len != SZ_1M && 3890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko len != SZ_64K && len != SZ_4K) { 3900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_debug("Bad size: %d\n", len); 3910720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -EINVAL; 3920720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 3930720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 3940720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3950720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!fl_table) { 3960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_debug("Null page table\n"); 3970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -EINVAL; 3980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 3990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 4000720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 401100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko if (len == SZ_16M || len == SZ_1M) { 402100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot = sh ? FL_SHARED : 0; 403100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot |= tex & 0x01 ? FL_BUFFERABLE : 0; 404100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot |= tex & 0x02 ? FL_CACHEABLE : 0; 405100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot |= tex & 0x04 ? FL_TEX0 : 0; 406100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko } else { 407100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot = sh ? SL_SHARED : 0; 408100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot |= tex & 0x01 ? SL_BUFFERABLE : 0; 409100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot |= tex & 0x02 ? SL_CACHEABLE : 0; 410100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot |= tex & 0x04 ? SL_TEX0 : 0; 411100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko } 412100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 4130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fl_offset = FL_OFFSET(va); /* Upper 12 bits */ 4140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */ 4150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_16M) { 4170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int i = 0; 4180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko for (i = 0; i < 16; i++) 4190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *(fl_pte+i) = (pa & 0xFF000000) | FL_SUPERSECTION | 4200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko FL_AP_READ | FL_AP_WRITE | FL_TYPE_SECT | 421100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko FL_SHARED | pgprot; 4220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 4230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_1M) 4250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *fl_pte = (pa & 0xFFF00000) | FL_AP_READ | FL_AP_WRITE | 426100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko FL_TYPE_SECT | FL_SHARED | pgprot; 4270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Need a 2nd level table */ 4290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if ((len == SZ_4K || len == SZ_64K) && (*fl_pte) == 0) { 4300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *sl; 431294b2dea83ba0a6d6034a7521bc62c317efab17bStepan Moskovchenko sl = (unsigned long *) __get_free_pages(GFP_ATOMIC, 4320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko get_order(SZ_4K)); 4330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!sl) { 4350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_debug("Could not allocate second level table\n"); 4360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -ENOMEM; 4370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 4380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 4390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko memset(sl, 0, SZ_4K); 4410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *fl_pte = ((((int)__pa(sl)) & FL_BASE_MASK) | FL_TYPE_TABLE); 4420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 4430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK)); 4450720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko sl_offset = SL_OFFSET(va); 4460720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko sl_pte = sl_table + sl_offset; 4470720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_4K) 4500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *sl_pte = (pa & SL_BASE_MASK_SMALL) | SL_AP0 | SL_AP1 | 451100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko SL_SHARED | SL_TYPE_SMALL | pgprot; 4520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_64K) { 4540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int i; 4550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko for (i = 0; i < 16; i++) 4570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *(sl_pte+i) = (pa & SL_BASE_MASK_LARGE) | SL_AP0 | 458100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko SL_AP1 | SL_SHARED | SL_TYPE_LARGE | pgprot; 4590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 4600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 46133069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko ret = __flush_iotlb(domain); 4620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail: 4630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_unlock_irqrestore(&msm_iommu_lock, flags); 4640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return ret; 4650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 4660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_unmap(struct iommu_domain *domain, unsigned long va, 4680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int order) 4690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 4700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv; 4710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long flags; 4720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *fl_table; 4730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *fl_pte; 4740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long fl_offset; 4750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *sl_table; 4760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *sl_pte; 4770720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long sl_offset; 4780720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko size_t len = 0x1000UL << order; 4790720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int i, ret = 0; 4800720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_lock_irqsave(&msm_iommu_lock, flags); 4820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko priv = domain->priv; 4840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!priv) { 4860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -ENODEV; 4870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 4880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 4890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fl_table = priv->pgtable; 4910720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4920720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len != SZ_16M && len != SZ_1M && 4930720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko len != SZ_64K && len != SZ_4K) { 4940720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_debug("Bad length: %d\n", len); 4950720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -EINVAL; 4960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 4970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 4980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!fl_table) { 5000720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_debug("Null page table\n"); 5010720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -EINVAL; 5020720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 5030720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 5040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5050720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fl_offset = FL_OFFSET(va); /* Upper 12 bits */ 5060720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */ 5070720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5080720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (*fl_pte == 0) { 5090720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_debug("First level PTE is 0\n"); 5100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -ENODEV; 5110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 5120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 5130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Unmap supersection */ 5150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_16M) 5160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko for (i = 0; i < 16; i++) 5170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *(fl_pte+i) = 0; 5180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_1M) 5200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *fl_pte = 0; 5210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK)); 5230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko sl_offset = SL_OFFSET(va); 5240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko sl_pte = sl_table + sl_offset; 5250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_64K) { 5270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko for (i = 0; i < 16; i++) 5280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *(sl_pte+i) = 0; 5290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 5300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_4K) 5320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *sl_pte = 0; 5330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_4K || len == SZ_64K) { 5350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int used = 0; 5360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko for (i = 0; i < NUM_SL_PTE; i++) 5380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (sl_table[i]) 5390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko used = 1; 5400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!used) { 5410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko free_page((unsigned long)sl_table); 5420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *fl_pte = 0; 5430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 5440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 5450720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 54633069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko ret = __flush_iotlb(domain); 5470720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail: 5480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_unlock_irqrestore(&msm_iommu_lock, flags); 5490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return ret; 5500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 5510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain, 5530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long va) 5540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 5550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv; 5560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_drvdata *iommu_drvdata; 5570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_drvdata *ctx_drvdata; 5580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned int par; 5590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long flags; 5600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko void __iomem *base; 5610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko phys_addr_t ret = 0; 5620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int ctx; 5630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_lock_irqsave(&msm_iommu_lock, flags); 5650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko priv = domain->priv; 5670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (list_empty(&priv->list_attached)) 5680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 5690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ctx_drvdata = list_entry(priv->list_attached.next, 5710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_drvdata, attached_elm); 5720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent); 5730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko base = iommu_drvdata->base; 5750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ctx = ctx_drvdata->num; 5760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 57741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko ret = __enable_clocks(iommu_drvdata); 57841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (ret) 57941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko goto fail; 58041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 5810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Invalidate context TLB */ 5820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_CTX_TLBIALL(base, ctx, 0); 5830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_V2PPR_VA(base, ctx, va >> V2Pxx_VA_SHIFT); 5840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko par = GET_PAR(base, ctx); 5860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* We are dealing with a supersection */ 5880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (GET_NOFAULT_SS(base, ctx)) 5890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = (par & 0xFF000000) | (va & 0x00FFFFFF); 5900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko else /* Upper 20 bits from PAR, lower 12 from VA */ 5910720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = (par & 0xFFFFF000) | (va & 0x00000FFF); 5920720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 59333069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko if (GET_FAULT(base, ctx)) 59433069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko ret = 0; 59533069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko 59641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko __disable_clocks(iommu_drvdata); 5970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail: 5980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_unlock_irqrestore(&msm_iommu_lock, flags); 5990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return ret; 6000720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 6010720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6020720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_domain_has_cap(struct iommu_domain *domain, 6030720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long cap) 6040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 6050720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return 0; 6060720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 6070720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6080720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void print_ctx_regs(void __iomem *base, int ctx) 6090720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 6100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned int fsr = GET_FSR(base, ctx); 6110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("FAR = %08x PAR = %08x\n", 6120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko GET_FAR(base, ctx), GET_PAR(base, ctx)); 6130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("FSR = %08x [%s%s%s%s%s%s%s%s%s%s]\n", fsr, 6140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x02) ? "TF " : "", 6150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x04) ? "AFF " : "", 6160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x08) ? "APF " : "", 6170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x10) ? "TLBMF " : "", 6180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x20) ? "HTWDEEF " : "", 6190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x40) ? "HTWSEEF " : "", 6200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x80) ? "MHF " : "", 6210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x10000) ? "SL " : "", 6220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x40000000) ? "SS " : "", 6230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x80000000) ? "MULTI " : ""); 6240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("FSYNR0 = %08x FSYNR1 = %08x\n", 6260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko GET_FSYNR0(base, ctx), GET_FSYNR1(base, ctx)); 6270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("TTBR0 = %08x TTBR1 = %08x\n", 6280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko GET_TTBR0(base, ctx), GET_TTBR1(base, ctx)); 6290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("SCTLR = %08x ACTLR = %08x\n", 6300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko GET_SCTLR(base, ctx), GET_ACTLR(base, ctx)); 6310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("PRRR = %08x NMRR = %08x\n", 6320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko GET_PRRR(base, ctx), GET_NMRR(base, ctx)); 6330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 6340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkoirqreturn_t msm_iommu_fault_handler(int irq, void *dev_id) 6360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 6370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_drvdata *drvdata = dev_id; 6380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko void __iomem *base; 63933069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko unsigned int fsr; 64041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko int ncb, i, ret; 6410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_lock(&msm_iommu_lock); 6430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!drvdata) { 6450720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("Invalid device ID in context interrupt handler\n"); 6460720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 6470720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 6480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko base = drvdata->base; 6500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("Unexpected IOMMU page fault!\n"); 6520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("base = %08x\n", (unsigned int) base); 6530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 65441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko ret = __enable_clocks(drvdata); 65541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (ret) 65641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko goto fail; 65741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 6580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ncb = GET_NCB(base)+1; 6590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko for (i = 0; i < ncb; i++) { 6600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fsr = GET_FSR(base, i); 6610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (fsr) { 6620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("Fault occurred in context %d.\n", i); 6630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("Interesting registers:\n"); 6640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko print_ctx_regs(base, i); 6650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_FSR(base, i, 0x4000000F); 6660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 6670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 66841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko __disable_clocks(drvdata); 6690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail: 6700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_unlock(&msm_iommu_lock); 6710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return 0; 6720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 6730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic struct iommu_ops msm_iommu_ops = { 6750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .domain_init = msm_iommu_domain_init, 6760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .domain_destroy = msm_iommu_domain_destroy, 6770720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .attach_dev = msm_iommu_attach_dev, 6780720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .detach_dev = msm_iommu_detach_dev, 6790720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .map = msm_iommu_map, 6800720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .unmap = msm_iommu_unmap, 6810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .iova_to_phys = msm_iommu_iova_to_phys, 6820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .domain_has_cap = msm_iommu_domain_has_cap 6830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}; 6840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 685100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenkostatic int __init get_tex_class(int icp, int ocp, int mt, int nos) 686100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko{ 687100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko int i = 0; 688100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko unsigned int prrr = 0; 689100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko unsigned int nmrr = 0; 690100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko int c_icp, c_ocp, c_mt, c_nos; 691100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 692100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko RCP15_PRRR(prrr); 693100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko RCP15_NMRR(nmrr); 694100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 695100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko for (i = 0; i < NUM_TEX_CLASS; i++) { 696100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko c_nos = PRRR_NOS(prrr, i); 697100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko c_mt = PRRR_MT(prrr, i); 698100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko c_icp = NMRR_ICP(nmrr, i); 699100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko c_ocp = NMRR_OCP(nmrr, i); 700100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 701100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko if (icp == c_icp && ocp == c_ocp && c_mt == mt && c_nos == nos) 702100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko return i; 703100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko } 704100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 705100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko return -ENODEV; 706100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko} 707100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 708100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenkostatic void __init setup_iommu_tex_classes(void) 709100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko{ 710100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko msm_iommu_tex_class[MSM_IOMMU_ATTR_NONCACHED] = 711100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko get_tex_class(CP_NONCACHED, CP_NONCACHED, MT_NORMAL, 1); 712100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 713100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WB_WA] = 714100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko get_tex_class(CP_WB_WA, CP_WB_WA, MT_NORMAL, 1); 715100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 716100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WB_NWA] = 717100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko get_tex_class(CP_WB_NWA, CP_WB_NWA, MT_NORMAL, 1); 718100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 719100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WT] = 720100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko get_tex_class(CP_WT, CP_WT, MT_NORMAL, 1); 721100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko} 722100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 723516cbc793eb4be5123289d067b54dfcdabeddb25Stepan Moskovchenkostatic int __init msm_iommu_init(void) 7240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 725100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko setup_iommu_tex_classes(); 7260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko register_iommu(&msm_iommu_ops); 7270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return 0; 7280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 7290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 7300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkosubsys_initcall(msm_iommu_init); 7310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 7320720d1f052dc1576396a39b327da6e60082c4efaStepan MoskovchenkoMODULE_LICENSE("GPL v2"); 7330720d1f052dc1576396a39b327da6e60082c4efaStepan MoskovchenkoMODULE_AUTHOR("Stepan Moskovchenko <stepanm@codeaurora.org>"); 734