141f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved. 20720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * 30720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * This program is free software; you can redistribute it and/or modify 40720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * it under the terms of the GNU General Public License version 2 and 50720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * only version 2 as published by the Free Software Foundation. 60720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * 70720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * This program is distributed in the hope that it will be useful, 80720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * but WITHOUT ANY WARRANTY; without even the implied warranty of 90720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * GNU General Public License for more details. 110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * 120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * You should have received a copy of the GNU General Public License 130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * along with this program; if not, write to the Free Software 140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * 02110-1301, USA. 160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko */ 170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/kernel.h> 200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/module.h> 210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/platform_device.h> 220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/errno.h> 230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/io.h> 240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/interrupt.h> 250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/list.h> 260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/spinlock.h> 270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/slab.h> 280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/iommu.h> 2941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko#include <linux/clk.h> 300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <asm/cacheflush.h> 320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <asm/sizes.h> 330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <mach/iommu_hw-8xxx.h> 350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <mach/iommu.h> 360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 37100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko#define MRC(reg, processor, op1, crn, crm, op2) \ 38100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko__asm__ __volatile__ ( \ 39100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko" mrc " #processor "," #op1 ", %0," #crn "," #crm "," #op2 "\n" \ 40100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko: "=r" (reg)) 41100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 42100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko#define RCP15_PRRR(reg) MRC(reg, p15, 0, c10, c2, 0) 43100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko#define RCP15_NMRR(reg) MRC(reg, p15, 0, c10, c2, 1) 44100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 4583427275546a6e36076d4f1a0545335b1bb2afc2Ohad Ben-Cohen/* bitmap of the page sizes currently supported */ 4683427275546a6e36076d4f1a0545335b1bb2afc2Ohad Ben-Cohen#define MSM_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M) 4783427275546a6e36076d4f1a0545335b1bb2afc2Ohad Ben-Cohen 48100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenkostatic int msm_iommu_tex_class[4]; 49100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 500720d1f052dc1576396a39b327da6e60082c4efaStepan MoskovchenkoDEFINE_SPINLOCK(msm_iommu_lock); 510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostruct msm_priv { 530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *pgtable; 540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct list_head list_attached; 550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}; 560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenkostatic int __enable_clocks(struct msm_iommu_drvdata *drvdata) 5841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko{ 5941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko int ret; 6041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 6141f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko ret = clk_enable(drvdata->pclk); 6241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (ret) 6341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko goto fail; 6441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 6541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (drvdata->clk) { 6641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko ret = clk_enable(drvdata->clk); 6741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (ret) 6841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko clk_disable(drvdata->pclk); 6941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko } 7041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenkofail: 7141f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko return ret; 7241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko} 7341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 7441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenkostatic void __disable_clocks(struct msm_iommu_drvdata *drvdata) 7541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko{ 7641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (drvdata->clk) 7741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko clk_disable(drvdata->clk); 7841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko clk_disable(drvdata->pclk); 7941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko} 8041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 8133069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenkostatic int __flush_iotlb(struct iommu_domain *domain) 820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv = domain->priv; 840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_drvdata *iommu_drvdata; 850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_drvdata *ctx_drvdata; 8633069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko int ret = 0; 870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#ifndef CONFIG_IOMMU_PGTABLES_L2 880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *fl_table = priv->pgtable; 890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int i; 900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 91f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko if (!list_empty(&priv->list_attached)) { 92f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko dmac_flush_range(fl_table, fl_table + SZ_16K); 930720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 94f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko for (i = 0; i < NUM_FL_PTE; i++) 95f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko if ((fl_table[i] & 0x03) == FL_TYPE_TABLE) { 96f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko void *sl_table = __va(fl_table[i] & 97f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko FL_BASE_MASK); 98f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko dmac_flush_range(sl_table, sl_table + SZ_4K); 99f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko } 100f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko } 1010720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#endif 1020720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1030720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko list_for_each_entry(ctx_drvdata, &priv->list_attached, attached_elm) { 1040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!ctx_drvdata->pdev || !ctx_drvdata->pdev->dev.parent) 1050720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko BUG(); 1060720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1070720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent); 10841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko BUG_ON(!iommu_drvdata); 10941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 11041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko ret = __enable_clocks(iommu_drvdata); 11141f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (ret) 11241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko goto fail; 11341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 1140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_CTX_TLBIALL(iommu_drvdata->base, ctx_drvdata->num, 0); 11541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko __disable_clocks(iommu_drvdata); 1160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 11741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenkofail: 11833069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko return ret; 1190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 1200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void __reset_context(void __iomem *base, int ctx) 1220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 1230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_BPRCOSH(base, ctx, 0); 1240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_BPRCISH(base, ctx, 0); 1250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_BPRCNSH(base, ctx, 0); 1260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_BPSHCFG(base, ctx, 0); 1270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_BPMTCFG(base, ctx, 0); 1280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_ACTLR(base, ctx, 0); 1290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_SCTLR(base, ctx, 0); 1300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_FSRRESTORE(base, ctx, 0); 1310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR0(base, ctx, 0); 1320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR1(base, ctx, 0); 1330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBCR(base, ctx, 0); 1340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_BFBCR(base, ctx, 0); 1350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_PAR(base, ctx, 0); 1360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_FAR(base, ctx, 0); 1370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_CTX_TLBIALL(base, ctx, 0); 1380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TLBFLPTER(base, ctx, 0); 1390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TLBSLPTER(base, ctx, 0); 1400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TLBLKCR(base, ctx, 0); 1410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_PRRR(base, ctx, 0); 1420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_NMRR(base, ctx, 0); 1430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 1440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1450720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void __program_context(void __iomem *base, int ctx, phys_addr_t pgtable) 1460720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 147100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko unsigned int prrr, nmrr; 1480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko __reset_context(base, ctx); 1490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Set up HTW mode */ 1510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* TLB miss configuration: perform HTW on miss */ 1520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TLBMCFG(base, ctx, 0x3); 1530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* V2P configuration: HTW for access */ 1550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_V2PCFG(base, ctx, 0x3); 1560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBCR(base, ctx, 0); 1580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR0_PA(base, ctx, (pgtable >> 14)); 1590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Invalidate the TLB for this context */ 1610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_CTX_TLBIALL(base, ctx, 0); 1620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Set interrupt number to "secure" interrupt */ 1640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_IRPTNDX(base, ctx, 0); 1650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Enable context fault interrupt */ 1670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_CFEIE(base, ctx, 1); 1680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Stall access on a context fault and let the handler deal with it */ 1700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_CFCFG(base, ctx, 1); 1710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Redirect all cacheable requests to L2 slave port. */ 1730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_RCISH(base, ctx, 1); 1740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_RCOSH(base, ctx, 1); 1750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_RCNSH(base, ctx, 1); 1760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1770720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Turn on TEX Remap */ 1780720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TRE(base, ctx, 1); 1790720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 180100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko /* Set TEX remap attributes */ 181100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko RCP15_PRRR(prrr); 182100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko RCP15_NMRR(nmrr); 183100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko SET_PRRR(base, ctx, prrr); 184100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko SET_NMRR(base, ctx, nmrr); 1850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Turn on BFB prefetch */ 1870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_BFBDFE(base, ctx, 1); 1880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#ifdef CONFIG_IOMMU_PGTABLES_L2 1900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Configure page tables as inner-cacheable and shareable to reduce 1910720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * the TLB miss penalty. 1920720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko */ 1930720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR0_SH(base, ctx, 1); 1940720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR1_SH(base, ctx, 1); 1950720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR0_NOS(base, ctx, 1); 1970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR1_NOS(base, ctx, 1); 1980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 1990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR0_IRGNH(base, ctx, 0); /* WB, WA */ 2000720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR0_IRGNL(base, ctx, 1); 2010720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2020720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR1_IRGNH(base, ctx, 0); /* WB, WA */ 2030720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR1_IRGNL(base, ctx, 1); 2040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2050720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR0_ORGN(base, ctx, 1); /* WB, WA */ 2060720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_TTBR1_ORGN(base, ctx, 1); /* WB, WA */ 2070720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#endif 2080720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2090720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Enable the MMU */ 2100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_M(base, ctx, 1); 2110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 2120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_domain_init(struct iommu_domain *domain) 2140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 2150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv = kzalloc(sizeof(*priv), GFP_KERNEL); 2160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!priv) 2180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail_nomem; 2190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko INIT_LIST_HEAD(&priv->list_attached); 2210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko priv->pgtable = (unsigned long *)__get_free_pages(GFP_KERNEL, 2220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko get_order(SZ_16K)); 2230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!priv->pgtable) 2250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail_nomem; 2260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko memset(priv->pgtable, 0, SZ_16K); 2280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko domain->priv = priv; 2290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return 0; 2300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail_nomem: 2320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko kfree(priv); 2330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return -ENOMEM; 2340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 2350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void msm_iommu_domain_destroy(struct iommu_domain *domain) 2370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 2380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv; 2390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long flags; 2400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *fl_table; 2410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int i; 2420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_lock_irqsave(&msm_iommu_lock, flags); 2440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko priv = domain->priv; 2450720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko domain->priv = NULL; 2460720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2470720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (priv) { 2480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fl_table = priv->pgtable; 2490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko for (i = 0; i < NUM_FL_PTE; i++) 2510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if ((fl_table[i] & 0x03) == FL_TYPE_TABLE) 2520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko free_page((unsigned long) __va(((fl_table[i]) & 2530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko FL_BASE_MASK))); 2540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko free_pages((unsigned long)priv->pgtable, get_order(SZ_16K)); 2560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko priv->pgtable = NULL; 2570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 2580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko kfree(priv); 2600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_unlock_irqrestore(&msm_iommu_lock, flags); 2610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 2620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) 2640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 2650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv; 2660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_dev *ctx_dev; 2670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_drvdata *iommu_drvdata; 2680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_drvdata *ctx_drvdata; 2690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_drvdata *tmp_drvdata; 2700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int ret = 0; 2710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long flags; 2720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_lock_irqsave(&msm_iommu_lock, flags); 2740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko priv = domain->priv; 2760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2770720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!priv || !dev) { 2780720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -EINVAL; 2790720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 2800720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 2810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko iommu_drvdata = dev_get_drvdata(dev->parent); 2830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ctx_drvdata = dev_get_drvdata(dev); 2840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ctx_dev = dev->platform_data; 2850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 2860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!iommu_drvdata || !ctx_drvdata || !ctx_dev) { 2870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -EINVAL; 2880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 2890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 2900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 29100d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko if (!list_empty(&ctx_drvdata->attached_elm)) { 29200d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko ret = -EBUSY; 29300d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko goto fail; 29400d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko } 29500d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko 2960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko list_for_each_entry(tmp_drvdata, &priv->list_attached, attached_elm) 2970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (tmp_drvdata == ctx_drvdata) { 2980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -EBUSY; 2990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 3000720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 3010720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 30241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko ret = __enable_clocks(iommu_drvdata); 30341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (ret) 30441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko goto fail; 30541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 3060720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko __program_context(iommu_drvdata->base, ctx_dev->num, 3070720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko __pa(priv->pgtable)); 3080720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 30941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko __disable_clocks(iommu_drvdata); 3100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko list_add(&(ctx_drvdata->attached_elm), &priv->list_attached); 31133069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko ret = __flush_iotlb(domain); 3120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail: 3140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_unlock_irqrestore(&msm_iommu_lock, flags); 3150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return ret; 3160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 3170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void msm_iommu_detach_dev(struct iommu_domain *domain, 3190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct device *dev) 3200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 3210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv; 3220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_dev *ctx_dev; 3230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_drvdata *iommu_drvdata; 3240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_drvdata *ctx_drvdata; 3250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long flags; 32633069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko int ret; 3270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_lock_irqsave(&msm_iommu_lock, flags); 3290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko priv = domain->priv; 3300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!priv || !dev) 3320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 3330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko iommu_drvdata = dev_get_drvdata(dev->parent); 3350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ctx_drvdata = dev_get_drvdata(dev); 3360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ctx_dev = dev->platform_data; 3370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!iommu_drvdata || !ctx_drvdata || !ctx_dev) 3390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 3400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 34133069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko ret = __flush_iotlb(domain); 34233069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko if (ret) 34333069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko goto fail; 34433069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko 34541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko ret = __enable_clocks(iommu_drvdata); 34641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (ret) 34741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko goto fail; 34841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 3490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko __reset_context(iommu_drvdata->base, ctx_dev->num); 35041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko __disable_clocks(iommu_drvdata); 3510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko list_del_init(&ctx_drvdata->attached_elm); 3520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail: 3540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_unlock_irqrestore(&msm_iommu_lock, flags); 3550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 3560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_map(struct iommu_domain *domain, unsigned long va, 3585009065d38c95455bd2d27c2838313e3dd0c5bc7Ohad Ben-Cohen phys_addr_t pa, size_t len, int prot) 3590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 3600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv; 3610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long flags; 3620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *fl_table; 3630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *fl_pte; 3640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long fl_offset; 3650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *sl_table; 3660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *sl_pte; 3670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long sl_offset; 368100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko unsigned int pgprot; 369100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko int ret = 0, tex, sh; 3700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_lock_irqsave(&msm_iommu_lock, flags); 3720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 373100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko sh = (prot & MSM_IOMMU_ATTR_SH) ? 1 : 0; 374100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko tex = msm_iommu_tex_class[prot & MSM_IOMMU_CP_MASK]; 375100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 376100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko if (tex < 0 || tex > NUM_TEX_CLASS - 1) { 377100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko ret = -EINVAL; 378100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko goto fail; 379100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko } 380100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 381100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko priv = domain->priv; 3820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!priv) { 3830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -EINVAL; 3840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 3850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 3860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fl_table = priv->pgtable; 3880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len != SZ_16M && len != SZ_1M && 3900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko len != SZ_64K && len != SZ_4K) { 3910720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_debug("Bad size: %d\n", len); 3920720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -EINVAL; 3930720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 3940720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 3950720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 3960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!fl_table) { 3970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_debug("Null page table\n"); 3980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -EINVAL; 3990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 4000720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 4010720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 402100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko if (len == SZ_16M || len == SZ_1M) { 403100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot = sh ? FL_SHARED : 0; 404100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot |= tex & 0x01 ? FL_BUFFERABLE : 0; 405100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot |= tex & 0x02 ? FL_CACHEABLE : 0; 406100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot |= tex & 0x04 ? FL_TEX0 : 0; 407100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko } else { 408100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot = sh ? SL_SHARED : 0; 409100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot |= tex & 0x01 ? SL_BUFFERABLE : 0; 410100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot |= tex & 0x02 ? SL_CACHEABLE : 0; 411100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko pgprot |= tex & 0x04 ? SL_TEX0 : 0; 412100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko } 413100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 4140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fl_offset = FL_OFFSET(va); /* Upper 12 bits */ 4150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */ 4160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_16M) { 4180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int i = 0; 4190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko for (i = 0; i < 16; i++) 4200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *(fl_pte+i) = (pa & 0xFF000000) | FL_SUPERSECTION | 4210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko FL_AP_READ | FL_AP_WRITE | FL_TYPE_SECT | 4222e8c8ba98376459e73d03a285f5d3406b630ea2dStepan Moskovchenko FL_SHARED | FL_NG | pgprot; 4230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 4240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_1M) 4262e8c8ba98376459e73d03a285f5d3406b630ea2dStepan Moskovchenko *fl_pte = (pa & 0xFFF00000) | FL_AP_READ | FL_AP_WRITE | FL_NG | 427100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko FL_TYPE_SECT | FL_SHARED | pgprot; 4280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Need a 2nd level table */ 4300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if ((len == SZ_4K || len == SZ_64K) && (*fl_pte) == 0) { 4310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *sl; 432294b2dea83ba0a6d6034a7521bc62c317efab17bStepan Moskovchenko sl = (unsigned long *) __get_free_pages(GFP_ATOMIC, 4330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko get_order(SZ_4K)); 4340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!sl) { 4360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_debug("Could not allocate second level table\n"); 4370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = -ENOMEM; 4380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 4390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 4400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko memset(sl, 0, SZ_4K); 4420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *fl_pte = ((((int)__pa(sl)) & FL_BASE_MASK) | FL_TYPE_TABLE); 4430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 4440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4450720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK)); 4460720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko sl_offset = SL_OFFSET(va); 4470720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko sl_pte = sl_table + sl_offset; 4480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_4K) 4512e8c8ba98376459e73d03a285f5d3406b630ea2dStepan Moskovchenko *sl_pte = (pa & SL_BASE_MASK_SMALL) | SL_AP0 | SL_AP1 | SL_NG | 452100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko SL_SHARED | SL_TYPE_SMALL | pgprot; 4530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_64K) { 4550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int i; 4560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko for (i = 0; i < 16; i++) 4580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *(sl_pte+i) = (pa & SL_BASE_MASK_LARGE) | SL_AP0 | 4592e8c8ba98376459e73d03a285f5d3406b630ea2dStepan Moskovchenko SL_NG | SL_AP1 | SL_SHARED | SL_TYPE_LARGE | pgprot; 4600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 4610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 46233069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko ret = __flush_iotlb(domain); 4630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail: 4640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_unlock_irqrestore(&msm_iommu_lock, flags); 4650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return ret; 4660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 4670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4685009065d38c95455bd2d27c2838313e3dd0c5bc7Ohad Ben-Cohenstatic size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long va, 4695009065d38c95455bd2d27c2838313e3dd0c5bc7Ohad Ben-Cohen size_t len) 4700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 4710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv; 4720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long flags; 4730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *fl_table; 4740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *fl_pte; 4750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long fl_offset; 4760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *sl_table; 4770720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long *sl_pte; 4780720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long sl_offset; 4790720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int i, ret = 0; 4800720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_lock_irqsave(&msm_iommu_lock, flags); 4820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko priv = domain->priv; 4840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 48505df1f3c2afaef5672627f2b7095f0d4c4dbc3a0Joerg Roedel if (!priv) 4860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 4870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fl_table = priv->pgtable; 4890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len != SZ_16M && len != SZ_1M && 4910720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko len != SZ_64K && len != SZ_4K) { 4920720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_debug("Bad length: %d\n", len); 4930720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 4940720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 4950720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 4960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!fl_table) { 4970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_debug("Null page table\n"); 4980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 4990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 5000720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5010720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fl_offset = FL_OFFSET(va); /* Upper 12 bits */ 5020720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */ 5030720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (*fl_pte == 0) { 5050720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_debug("First level PTE is 0\n"); 5060720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 5070720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 5080720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5090720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Unmap supersection */ 5100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_16M) 5110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko for (i = 0; i < 16; i++) 5120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *(fl_pte+i) = 0; 5130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_1M) 5150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *fl_pte = 0; 5160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK)); 5180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko sl_offset = SL_OFFSET(va); 5190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko sl_pte = sl_table + sl_offset; 5200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_64K) { 5220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko for (i = 0; i < 16; i++) 5230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *(sl_pte+i) = 0; 5240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 5250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_4K) 5270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *sl_pte = 0; 5280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (len == SZ_4K || len == SZ_64K) { 5300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int used = 0; 5310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko for (i = 0; i < NUM_SL_PTE; i++) 5330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (sl_table[i]) 5340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko used = 1; 5350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!used) { 5360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko free_page((unsigned long)sl_table); 5370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *fl_pte = 0; 5380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 5390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 5400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 54133069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko ret = __flush_iotlb(domain); 5429e28547f887c3040747d8422e1feef79bea93bd8Ohad Ben-Cohen 5430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail: 5440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_unlock_irqrestore(&msm_iommu_lock, flags); 5455009065d38c95455bd2d27c2838313e3dd0c5bc7Ohad Ben-Cohen 5465009065d38c95455bd2d27c2838313e3dd0c5bc7Ohad Ben-Cohen /* the IOMMU API requires us to return how many bytes were unmapped */ 5475009065d38c95455bd2d27c2838313e3dd0c5bc7Ohad Ben-Cohen len = ret ? 0 : len; 5485009065d38c95455bd2d27c2838313e3dd0c5bc7Ohad Ben-Cohen return len; 5490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 5500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain, 5520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long va) 5530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 5540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_priv *priv; 5550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_drvdata *iommu_drvdata; 5560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_drvdata *ctx_drvdata; 5570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned int par; 5580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long flags; 5590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko void __iomem *base; 5600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko phys_addr_t ret = 0; 5610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko int ctx; 5620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_lock_irqsave(&msm_iommu_lock, flags); 5640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko priv = domain->priv; 5660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (list_empty(&priv->list_attached)) 5670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 5680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ctx_drvdata = list_entry(priv->list_attached.next, 5700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_ctx_drvdata, attached_elm); 5710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent); 5720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko base = iommu_drvdata->base; 5740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ctx = ctx_drvdata->num; 5750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 57641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko ret = __enable_clocks(iommu_drvdata); 57741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (ret) 57841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko goto fail; 57941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 5800720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* Invalidate context TLB */ 5810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_CTX_TLBIALL(base, ctx, 0); 582b0e7808d548ea1d857216d31d63078411203a116Stepan Moskovchenko SET_V2PPR(base, ctx, va & V2Pxx_VA); 5830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko par = GET_PAR(base, ctx); 5850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 5860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko /* We are dealing with a supersection */ 5870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (GET_NOFAULT_SS(base, ctx)) 5880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = (par & 0xFF000000) | (va & 0x00FFFFFF); 5890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko else /* Upper 20 bits from PAR, lower 12 from VA */ 5900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko ret = (par & 0xFFFFF000) | (va & 0x00000FFF); 5910720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 59233069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko if (GET_FAULT(base, ctx)) 59333069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko ret = 0; 59433069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko 59541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko __disable_clocks(iommu_drvdata); 5960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail: 5970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_unlock_irqrestore(&msm_iommu_lock, flags); 5980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return ret; 5990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 6000720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6010720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_domain_has_cap(struct iommu_domain *domain, 6020720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned long cap) 6030720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 6040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return 0; 6050720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 6060720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6070720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void print_ctx_regs(void __iomem *base, int ctx) 6080720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 6090720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko unsigned int fsr = GET_FSR(base, ctx); 6100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("FAR = %08x PAR = %08x\n", 6110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko GET_FAR(base, ctx), GET_PAR(base, ctx)); 6120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("FSR = %08x [%s%s%s%s%s%s%s%s%s%s]\n", fsr, 6130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x02) ? "TF " : "", 6140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x04) ? "AFF " : "", 6150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x08) ? "APF " : "", 6160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x10) ? "TLBMF " : "", 6170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x20) ? "HTWDEEF " : "", 6180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x40) ? "HTWSEEF " : "", 6190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x80) ? "MHF " : "", 6200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x10000) ? "SL " : "", 6210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x40000000) ? "SS " : "", 6220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko (fsr & 0x80000000) ? "MULTI " : ""); 6230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("FSYNR0 = %08x FSYNR1 = %08x\n", 6250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko GET_FSYNR0(base, ctx), GET_FSYNR1(base, ctx)); 6260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("TTBR0 = %08x TTBR1 = %08x\n", 6270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko GET_TTBR0(base, ctx), GET_TTBR1(base, ctx)); 6280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("SCTLR = %08x ACTLR = %08x\n", 6290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko GET_SCTLR(base, ctx), GET_ACTLR(base, ctx)); 6300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("PRRR = %08x NMRR = %08x\n", 6310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko GET_PRRR(base, ctx), GET_NMRR(base, ctx)); 6320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 6330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkoirqreturn_t msm_iommu_fault_handler(int irq, void *dev_id) 6350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 6360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko struct msm_iommu_drvdata *drvdata = dev_id; 6370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko void __iomem *base; 63833069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko unsigned int fsr; 639a43d8c101eb71bf4527dd7f36a34a5a502894f38Stepan Moskovchenko int i, ret; 6400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_lock(&msm_iommu_lock); 6420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (!drvdata) { 6440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("Invalid device ID in context interrupt handler\n"); 6450720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko goto fail; 6460720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 6470720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko base = drvdata->base; 6490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("Unexpected IOMMU page fault!\n"); 6510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("base = %08x\n", (unsigned int) base); 6520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 65341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko ret = __enable_clocks(drvdata); 65441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko if (ret) 65541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko goto fail; 65641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko 657a43d8c101eb71bf4527dd7f36a34a5a502894f38Stepan Moskovchenko for (i = 0; i < drvdata->ncb; i++) { 6580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko fsr = GET_FSR(base, i); 6590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko if (fsr) { 6600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("Fault occurred in context %d.\n", i); 6610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko pr_err("Interesting registers:\n"); 6620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko print_ctx_regs(base, i); 6630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko SET_FSR(base, i, 0x4000000F); 6640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 6650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko } 66641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko __disable_clocks(drvdata); 6670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail: 6680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko spin_unlock(&msm_iommu_lock); 6690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return 0; 6700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 6710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 6720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic struct iommu_ops msm_iommu_ops = { 6730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .domain_init = msm_iommu_domain_init, 6740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .domain_destroy = msm_iommu_domain_destroy, 6750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .attach_dev = msm_iommu_attach_dev, 6760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .detach_dev = msm_iommu_detach_dev, 6770720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .map = msm_iommu_map, 6780720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .unmap = msm_iommu_unmap, 6790720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko .iova_to_phys = msm_iommu_iova_to_phys, 68083427275546a6e36076d4f1a0545335b1bb2afc2Ohad Ben-Cohen .domain_has_cap = msm_iommu_domain_has_cap, 68183427275546a6e36076d4f1a0545335b1bb2afc2Ohad Ben-Cohen .pgsize_bitmap = MSM_IOMMU_PGSIZES, 6820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}; 6830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 684100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenkostatic int __init get_tex_class(int icp, int ocp, int mt, int nos) 685100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko{ 686100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko int i = 0; 687100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko unsigned int prrr = 0; 688100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko unsigned int nmrr = 0; 689100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko int c_icp, c_ocp, c_mt, c_nos; 690100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 691100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko RCP15_PRRR(prrr); 692100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko RCP15_NMRR(nmrr); 693100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 694100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko for (i = 0; i < NUM_TEX_CLASS; i++) { 695100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko c_nos = PRRR_NOS(prrr, i); 696100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko c_mt = PRRR_MT(prrr, i); 697100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko c_icp = NMRR_ICP(nmrr, i); 698100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko c_ocp = NMRR_OCP(nmrr, i); 699100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 700100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko if (icp == c_icp && ocp == c_ocp && c_mt == mt && c_nos == nos) 701100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko return i; 702100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko } 703100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 704100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko return -ENODEV; 705100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko} 706100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 707100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenkostatic void __init setup_iommu_tex_classes(void) 708100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko{ 709100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko msm_iommu_tex_class[MSM_IOMMU_ATTR_NONCACHED] = 710100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko get_tex_class(CP_NONCACHED, CP_NONCACHED, MT_NORMAL, 1); 711100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 712100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WB_WA] = 713100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko get_tex_class(CP_WB_WA, CP_WB_WA, MT_NORMAL, 1); 714100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 715100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WB_NWA] = 716100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko get_tex_class(CP_WB_NWA, CP_WB_NWA, MT_NORMAL, 1); 717100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 718100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WT] = 719100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko get_tex_class(CP_WT, CP_WT, MT_NORMAL, 1); 720100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko} 721100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko 722516cbc793eb4be5123289d067b54dfcdabeddb25Stepan Moskovchenkostatic int __init msm_iommu_init(void) 7230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{ 724100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko setup_iommu_tex_classes(); 72585eebbc5e4633e6eee10bdddd00d175daadd5841Joerg Roedel bus_set_iommu(&platform_bus_type, &msm_iommu_ops); 7260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko return 0; 7270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko} 7280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 7290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkosubsys_initcall(msm_iommu_init); 7300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko 7310720d1f052dc1576396a39b327da6e60082c4efaStepan MoskovchenkoMODULE_LICENSE("GPL v2"); 7320720d1f052dc1576396a39b327da6e60082c4efaStepan MoskovchenkoMODULE_AUTHOR("Stepan Moskovchenko <stepanm@codeaurora.org>"); 733