msm_iommu.c revision 9e28547f887c3040747d8422e1feef79bea93bd8
141f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved.
20720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *
30720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * This program is free software; you can redistribute it and/or modify
40720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * it under the terms of the GNU General Public License version 2 and
50720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * only version 2 as published by the Free Software Foundation.
60720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *
70720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * This program is distributed in the hope that it will be useful,
80720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * but WITHOUT ANY WARRANTY; without even the implied warranty of
90720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * GNU General Public License for more details.
110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *
120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * You should have received a copy of the GNU General Public License
130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * along with this program; if not, write to the Free Software
140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * 02110-1301, USA.
160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko */
170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/kernel.h>
200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/module.h>
210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/platform_device.h>
220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/errno.h>
230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/io.h>
240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/interrupt.h>
250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/list.h>
260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/spinlock.h>
270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/slab.h>
280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/iommu.h>
2941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko#include <linux/clk.h>
300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <asm/cacheflush.h>
320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <asm/sizes.h>
330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <mach/iommu_hw-8xxx.h>
350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <mach/iommu.h>
360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
37100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko#define MRC(reg, processor, op1, crn, crm, op2)				\
38100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko__asm__ __volatile__ (							\
39100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko"   mrc   "   #processor "," #op1 ", %0,"  #crn "," #crm "," #op2 "\n"  \
40100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko: "=r" (reg))
41100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
42100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko#define RCP15_PRRR(reg)		MRC(reg, p15, 0, c10, c2, 0)
43100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko#define RCP15_NMRR(reg)		MRC(reg, p15, 0, c10, c2, 1)
44100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
45100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenkostatic int msm_iommu_tex_class[4];
46100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
470720d1f052dc1576396a39b327da6e60082c4efaStepan MoskovchenkoDEFINE_SPINLOCK(msm_iommu_lock);
480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostruct msm_priv {
500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long *pgtable;
510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct list_head list_attached;
520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko};
530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenkostatic int __enable_clocks(struct msm_iommu_drvdata *drvdata)
5541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko{
5641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	int ret;
5741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko
5841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	ret = clk_enable(drvdata->pclk);
5941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	if (ret)
6041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		goto fail;
6141f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko
6241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	if (drvdata->clk) {
6341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		ret = clk_enable(drvdata->clk);
6441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		if (ret)
6541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko			clk_disable(drvdata->pclk);
6641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	}
6741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenkofail:
6841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	return ret;
6941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko}
7041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko
7141f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenkostatic void __disable_clocks(struct msm_iommu_drvdata *drvdata)
7241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko{
7341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	if (drvdata->clk)
7441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		clk_disable(drvdata->clk);
7541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	clk_disable(drvdata->pclk);
7641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko}
7741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko
7833069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenkostatic int __flush_iotlb(struct iommu_domain *domain)
790720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
800720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_priv *priv = domain->priv;
810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_drvdata *iommu_drvdata;
820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_ctx_drvdata *ctx_drvdata;
8333069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko	int ret = 0;
840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#ifndef CONFIG_IOMMU_PGTABLES_L2
850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long *fl_table = priv->pgtable;
860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	int i;
870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
88f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko	if (!list_empty(&priv->list_attached)) {
89f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko		dmac_flush_range(fl_table, fl_table + SZ_16K);
900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
91f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko		for (i = 0; i < NUM_FL_PTE; i++)
92f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko			if ((fl_table[i] & 0x03) == FL_TYPE_TABLE) {
93f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko				void *sl_table = __va(fl_table[i] &
94f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko								FL_BASE_MASK);
95f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko				dmac_flush_range(sl_table, sl_table + SZ_4K);
96f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko			}
97f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko	}
980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#endif
990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1000720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	list_for_each_entry(ctx_drvdata, &priv->list_attached, attached_elm) {
1010720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		if (!ctx_drvdata->pdev || !ctx_drvdata->pdev->dev.parent)
1020720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			BUG();
1030720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent);
10541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		BUG_ON(!iommu_drvdata);
10641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko
10741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		ret = __enable_clocks(iommu_drvdata);
10841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		if (ret)
10941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko			goto fail;
11041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko
1110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		SET_CTX_TLBIALL(iommu_drvdata->base, ctx_drvdata->num, 0);
11241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		__disable_clocks(iommu_drvdata);
1130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
11441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenkofail:
11533069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko	return ret;
1160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
1170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void __reset_context(void __iomem *base, int ctx)
1190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
1200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_BPRCOSH(base, ctx, 0);
1210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_BPRCISH(base, ctx, 0);
1220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_BPRCNSH(base, ctx, 0);
1230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_BPSHCFG(base, ctx, 0);
1240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_BPMTCFG(base, ctx, 0);
1250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_ACTLR(base, ctx, 0);
1260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_SCTLR(base, ctx, 0);
1270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_FSRRESTORE(base, ctx, 0);
1280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR0(base, ctx, 0);
1290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR1(base, ctx, 0);
1300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBCR(base, ctx, 0);
1310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_BFBCR(base, ctx, 0);
1320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_PAR(base, ctx, 0);
1330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_FAR(base, ctx, 0);
1340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_CTX_TLBIALL(base, ctx, 0);
1350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TLBFLPTER(base, ctx, 0);
1360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TLBSLPTER(base, ctx, 0);
1370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TLBLKCR(base, ctx, 0);
1380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_PRRR(base, ctx, 0);
1390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_NMRR(base, ctx, 0);
1400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
1410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void __program_context(void __iomem *base, int ctx, phys_addr_t pgtable)
1430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
144100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	unsigned int prrr, nmrr;
1450720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	__reset_context(base, ctx);
1460720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1470720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Set up HTW mode */
1480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* TLB miss configuration: perform HTW on miss */
1490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TLBMCFG(base, ctx, 0x3);
1500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* V2P configuration: HTW for access */
1520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_V2PCFG(base, ctx, 0x3);
1530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBCR(base, ctx, 0);
1550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR0_PA(base, ctx, (pgtable >> 14));
1560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Invalidate the TLB for this context */
1580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_CTX_TLBIALL(base, ctx, 0);
1590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Set interrupt number to "secure" interrupt */
1610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_IRPTNDX(base, ctx, 0);
1620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Enable context fault interrupt */
1640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_CFEIE(base, ctx, 1);
1650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Stall access on a context fault and let the handler deal with it */
1670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_CFCFG(base, ctx, 1);
1680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Redirect all cacheable requests to L2 slave port. */
1700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_RCISH(base, ctx, 1);
1710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_RCOSH(base, ctx, 1);
1720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_RCNSH(base, ctx, 1);
1730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Turn on TEX Remap */
1750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TRE(base, ctx, 1);
1760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
177100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	/* Set TEX remap attributes */
178100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	RCP15_PRRR(prrr);
179100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	RCP15_NMRR(nmrr);
180100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	SET_PRRR(base, ctx, prrr);
181100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	SET_NMRR(base, ctx, nmrr);
1820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Turn on BFB prefetch */
1840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_BFBDFE(base, ctx, 1);
1850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#ifdef CONFIG_IOMMU_PGTABLES_L2
1870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Configure page tables as inner-cacheable and shareable to reduce
1880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	 * the TLB miss penalty.
1890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	 */
1900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR0_SH(base, ctx, 1);
1910720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR1_SH(base, ctx, 1);
1920720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1930720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR0_NOS(base, ctx, 1);
1940720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR1_NOS(base, ctx, 1);
1950720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR0_IRGNH(base, ctx, 0); /* WB, WA */
1970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR0_IRGNL(base, ctx, 1);
1980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR1_IRGNH(base, ctx, 0); /* WB, WA */
2000720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR1_IRGNL(base, ctx, 1);
2010720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2020720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR0_ORGN(base, ctx, 1); /* WB, WA */
2030720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR1_ORGN(base, ctx, 1); /* WB, WA */
2040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#endif
2050720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2060720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Enable the MMU */
2070720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_M(base, ctx, 1);
2080720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
2090720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_domain_init(struct iommu_domain *domain)
2110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
2120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_priv *priv = kzalloc(sizeof(*priv), GFP_KERNEL);
2130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (!priv)
2150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail_nomem;
2160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	INIT_LIST_HEAD(&priv->list_attached);
2180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	priv->pgtable = (unsigned long *)__get_free_pages(GFP_KERNEL,
2190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko							  get_order(SZ_16K));
2200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (!priv->pgtable)
2220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail_nomem;
2230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	memset(priv->pgtable, 0, SZ_16K);
2250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	domain->priv = priv;
2260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	return 0;
2270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail_nomem:
2290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	kfree(priv);
2300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	return -ENOMEM;
2310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
2320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void msm_iommu_domain_destroy(struct iommu_domain *domain)
2340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
2350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_priv *priv;
2360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long flags;
2370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long *fl_table;
2380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	int i;
2390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_lock_irqsave(&msm_iommu_lock, flags);
2410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	priv = domain->priv;
2420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	domain->priv = NULL;
2430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (priv) {
2450720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		fl_table = priv->pgtable;
2460720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2470720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		for (i = 0; i < NUM_FL_PTE; i++)
2480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			if ((fl_table[i] & 0x03) == FL_TYPE_TABLE)
2490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko				free_page((unsigned long) __va(((fl_table[i]) &
2500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko								FL_BASE_MASK)));
2510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		free_pages((unsigned long)priv->pgtable, get_order(SZ_16K));
2530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		priv->pgtable = NULL;
2540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
2550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	kfree(priv);
2570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_unlock_irqrestore(&msm_iommu_lock, flags);
2580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
2590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
2610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
2620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_priv *priv;
2630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_ctx_dev *ctx_dev;
2640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_drvdata *iommu_drvdata;
2650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_ctx_drvdata *ctx_drvdata;
2660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_ctx_drvdata *tmp_drvdata;
2670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	int ret = 0;
2680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long flags;
2690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_lock_irqsave(&msm_iommu_lock, flags);
2710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	priv = domain->priv;
2730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (!priv || !dev) {
2750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		ret = -EINVAL;
2760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
2770720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
2780720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2790720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	iommu_drvdata = dev_get_drvdata(dev->parent);
2800720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	ctx_drvdata = dev_get_drvdata(dev);
2810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	ctx_dev = dev->platform_data;
2820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (!iommu_drvdata || !ctx_drvdata || !ctx_dev) {
2840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		ret = -EINVAL;
2850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
2860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
2870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
28800d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko	if (!list_empty(&ctx_drvdata->attached_elm)) {
28900d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko		ret = -EBUSY;
29000d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko		goto fail;
29100d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko	}
29200d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko
2930720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	list_for_each_entry(tmp_drvdata, &priv->list_attached, attached_elm)
2940720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		if (tmp_drvdata == ctx_drvdata) {
2950720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			ret = -EBUSY;
2960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			goto fail;
2970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		}
2980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
29941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	ret = __enable_clocks(iommu_drvdata);
30041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	if (ret)
30141f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		goto fail;
30241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko
3030720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	__program_context(iommu_drvdata->base, ctx_dev->num,
3040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			  __pa(priv->pgtable));
3050720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
30641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	__disable_clocks(iommu_drvdata);
3070720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	list_add(&(ctx_drvdata->attached_elm), &priv->list_attached);
30833069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko	ret = __flush_iotlb(domain);
3090720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail:
3110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_unlock_irqrestore(&msm_iommu_lock, flags);
3120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	return ret;
3130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
3140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void msm_iommu_detach_dev(struct iommu_domain *domain,
3160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko				 struct device *dev)
3170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
3180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_priv *priv;
3190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_ctx_dev *ctx_dev;
3200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_drvdata *iommu_drvdata;
3210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_ctx_drvdata *ctx_drvdata;
3220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long flags;
32333069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko	int ret;
3240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_lock_irqsave(&msm_iommu_lock, flags);
3260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	priv = domain->priv;
3270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (!priv || !dev)
3290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
3300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	iommu_drvdata = dev_get_drvdata(dev->parent);
3320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	ctx_drvdata = dev_get_drvdata(dev);
3330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	ctx_dev = dev->platform_data;
3340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (!iommu_drvdata || !ctx_drvdata || !ctx_dev)
3360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
3370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
33833069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko	ret = __flush_iotlb(domain);
33933069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko	if (ret)
34033069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko		goto fail;
34133069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko
34241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	ret = __enable_clocks(iommu_drvdata);
34341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	if (ret)
34441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		goto fail;
34541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko
3460720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	__reset_context(iommu_drvdata->base, ctx_dev->num);
34741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	__disable_clocks(iommu_drvdata);
3480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	list_del_init(&ctx_drvdata->attached_elm);
3490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail:
3510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_unlock_irqrestore(&msm_iommu_lock, flags);
3520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
3530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_map(struct iommu_domain *domain, unsigned long va,
3550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			 phys_addr_t pa, int order, int prot)
3560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
3570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_priv *priv;
3580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long flags;
3590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long *fl_table;
3600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long *fl_pte;
3610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long fl_offset;
3620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long *sl_table;
3630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long *sl_pte;
3640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long sl_offset;
365100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	unsigned int pgprot;
3660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	size_t len = 0x1000UL << order;
367100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	int ret = 0, tex, sh;
3680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_lock_irqsave(&msm_iommu_lock, flags);
3700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
371100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	sh = (prot & MSM_IOMMU_ATTR_SH) ? 1 : 0;
372100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	tex = msm_iommu_tex_class[prot & MSM_IOMMU_CP_MASK];
373100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
374100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	if (tex < 0 || tex > NUM_TEX_CLASS - 1) {
375100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		ret = -EINVAL;
376100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		goto fail;
377100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	}
378100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
379100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	priv = domain->priv;
3800720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (!priv) {
3810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		ret = -EINVAL;
3820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
3830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
3840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	fl_table = priv->pgtable;
3860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (len != SZ_16M && len != SZ_1M &&
3880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	    len != SZ_64K && len != SZ_4K) {
3890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		pr_debug("Bad size: %d\n", len);
3900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		ret = -EINVAL;
3910720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
3920720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
3930720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3940720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (!fl_table) {
3950720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		pr_debug("Null page table\n");
3960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		ret = -EINVAL;
3970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
3980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
3990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
400100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	if (len == SZ_16M || len == SZ_1M) {
401100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		pgprot = sh ? FL_SHARED : 0;
402100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		pgprot |= tex & 0x01 ? FL_BUFFERABLE : 0;
403100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		pgprot |= tex & 0x02 ? FL_CACHEABLE : 0;
404100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		pgprot |= tex & 0x04 ? FL_TEX0 : 0;
405100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	} else	{
406100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		pgprot = sh ? SL_SHARED : 0;
407100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		pgprot |= tex & 0x01 ? SL_BUFFERABLE : 0;
408100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		pgprot |= tex & 0x02 ? SL_CACHEABLE : 0;
409100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		pgprot |= tex & 0x04 ? SL_TEX0 : 0;
410100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	}
411100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
4120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	fl_offset = FL_OFFSET(va);	/* Upper 12 bits */
4130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	fl_pte = fl_table + fl_offset;	/* int pointers, 4 bytes */
4140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (len == SZ_16M) {
4160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		int i = 0;
4170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		for (i = 0; i < 16; i++)
4180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			*(fl_pte+i) = (pa & 0xFF000000) | FL_SUPERSECTION |
4190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko				  FL_AP_READ | FL_AP_WRITE | FL_TYPE_SECT |
4202e8c8ba98376459e73d03a285f5d3406b630ea2dStepan Moskovchenko				  FL_SHARED | FL_NG | pgprot;
4210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
4220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (len == SZ_1M)
4242e8c8ba98376459e73d03a285f5d3406b630ea2dStepan Moskovchenko		*fl_pte = (pa & 0xFFF00000) | FL_AP_READ | FL_AP_WRITE | FL_NG |
425100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko					    FL_TYPE_SECT | FL_SHARED | pgprot;
4260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Need a 2nd level table */
4280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if ((len == SZ_4K || len == SZ_64K) && (*fl_pte) == 0) {
4290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		unsigned long *sl;
430294b2dea83ba0a6d6034a7521bc62c317efab17bStepan Moskovchenko		sl = (unsigned long *) __get_free_pages(GFP_ATOMIC,
4310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko							get_order(SZ_4K));
4320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		if (!sl) {
4340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			pr_debug("Could not allocate second level table\n");
4350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			ret = -ENOMEM;
4360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			goto fail;
4370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		}
4380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		memset(sl, 0, SZ_4K);
4400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		*fl_pte = ((((int)__pa(sl)) & FL_BASE_MASK) | FL_TYPE_TABLE);
4410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
4420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK));
4440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	sl_offset = SL_OFFSET(va);
4450720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	sl_pte = sl_table + sl_offset;
4460720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4470720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (len == SZ_4K)
4492e8c8ba98376459e73d03a285f5d3406b630ea2dStepan Moskovchenko		*sl_pte = (pa & SL_BASE_MASK_SMALL) | SL_AP0 | SL_AP1 | SL_NG |
450100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko					  SL_SHARED | SL_TYPE_SMALL | pgprot;
4510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (len == SZ_64K) {
4530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		int i;
4540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		for (i = 0; i < 16; i++)
4560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			*(sl_pte+i) = (pa & SL_BASE_MASK_LARGE) | SL_AP0 |
4572e8c8ba98376459e73d03a285f5d3406b630ea2dStepan Moskovchenko			    SL_NG | SL_AP1 | SL_SHARED | SL_TYPE_LARGE | pgprot;
4580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
4590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
46033069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko	ret = __flush_iotlb(domain);
4610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail:
4620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_unlock_irqrestore(&msm_iommu_lock, flags);
4630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	return ret;
4640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
4650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_unmap(struct iommu_domain *domain, unsigned long va,
4670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			    int order)
4680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
4690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_priv *priv;
4700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long flags;
4710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long *fl_table;
4720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long *fl_pte;
4730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long fl_offset;
4740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long *sl_table;
4750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long *sl_pte;
4760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long sl_offset;
4770720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	size_t len = 0x1000UL << order;
4780720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	int i, ret = 0;
4790720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4800720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_lock_irqsave(&msm_iommu_lock, flags);
4810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	priv = domain->priv;
4830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (!priv) {
4850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		ret = -ENODEV;
4860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
4870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
4880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	fl_table = priv->pgtable;
4900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4910720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (len != SZ_16M && len != SZ_1M &&
4920720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	    len != SZ_64K && len != SZ_4K) {
4930720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		pr_debug("Bad length: %d\n", len);
4940720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		ret = -EINVAL;
4950720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
4960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
4970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (!fl_table) {
4990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		pr_debug("Null page table\n");
5000720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		ret = -EINVAL;
5010720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
5020720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
5030720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	fl_offset = FL_OFFSET(va);	/* Upper 12 bits */
5050720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	fl_pte = fl_table + fl_offset;	/* int pointers, 4 bytes */
5060720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5070720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (*fl_pte == 0) {
5080720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		pr_debug("First level PTE is 0\n");
5090720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		ret = -ENODEV;
5100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
5110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
5120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Unmap supersection */
5140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (len == SZ_16M)
5150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		for (i = 0; i < 16; i++)
5160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			*(fl_pte+i) = 0;
5170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (len == SZ_1M)
5190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		*fl_pte = 0;
5200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK));
5220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	sl_offset = SL_OFFSET(va);
5230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	sl_pte = sl_table + sl_offset;
5240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (len == SZ_64K) {
5260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		for (i = 0; i < 16; i++)
5270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			*(sl_pte+i) = 0;
5280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
5290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (len == SZ_4K)
5310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		*sl_pte = 0;
5320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (len == SZ_4K || len == SZ_64K) {
5340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		int used = 0;
5350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		for (i = 0; i < NUM_SL_PTE; i++)
5370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			if (sl_table[i])
5380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko				used = 1;
5390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		if (!used) {
5400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			free_page((unsigned long)sl_table);
5410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			*fl_pte = 0;
5420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		}
5430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
5440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
54533069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko	ret = __flush_iotlb(domain);
5469e28547f887c3040747d8422e1feef79bea93bd8Ohad Ben-Cohen
5479e28547f887c3040747d8422e1feef79bea93bd8Ohad Ben-Cohen	/*
5489e28547f887c3040747d8422e1feef79bea93bd8Ohad Ben-Cohen	 * the IOMMU API requires us to return the order of the unmapped
5499e28547f887c3040747d8422e1feef79bea93bd8Ohad Ben-Cohen	 * page (on success).
5509e28547f887c3040747d8422e1feef79bea93bd8Ohad Ben-Cohen	 */
5519e28547f887c3040747d8422e1feef79bea93bd8Ohad Ben-Cohen	if (!ret)
5529e28547f887c3040747d8422e1feef79bea93bd8Ohad Ben-Cohen		ret = order;
5530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail:
5540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_unlock_irqrestore(&msm_iommu_lock, flags);
5550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	return ret;
5560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
5570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain,
5590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko					  unsigned long va)
5600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
5610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_priv *priv;
5620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_drvdata *iommu_drvdata;
5630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_ctx_drvdata *ctx_drvdata;
5640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned int par;
5650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long flags;
5660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	void __iomem *base;
5670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	phys_addr_t ret = 0;
5680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	int ctx;
5690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_lock_irqsave(&msm_iommu_lock, flags);
5710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	priv = domain->priv;
5730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (list_empty(&priv->list_attached))
5740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
5750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	ctx_drvdata = list_entry(priv->list_attached.next,
5770720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko				 struct msm_iommu_ctx_drvdata, attached_elm);
5780720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent);
5790720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5800720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	base = iommu_drvdata->base;
5810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	ctx = ctx_drvdata->num;
5820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
58341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	ret = __enable_clocks(iommu_drvdata);
58441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	if (ret)
58541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		goto fail;
58641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko
5870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Invalidate context TLB */
5880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_CTX_TLBIALL(base, ctx, 0);
589b0e7808d548ea1d857216d31d63078411203a116Stepan Moskovchenko	SET_V2PPR(base, ctx, va & V2Pxx_VA);
5900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5910720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	par = GET_PAR(base, ctx);
5920720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5930720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* We are dealing with a supersection */
5940720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (GET_NOFAULT_SS(base, ctx))
5950720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		ret = (par & 0xFF000000) | (va & 0x00FFFFFF);
5960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	else	/* Upper 20 bits from PAR, lower 12 from VA */
5970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		ret = (par & 0xFFFFF000) | (va & 0x00000FFF);
5980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
59933069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko	if (GET_FAULT(base, ctx))
60033069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko		ret = 0;
60133069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko
60241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	__disable_clocks(iommu_drvdata);
6030720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail:
6040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_unlock_irqrestore(&msm_iommu_lock, flags);
6050720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	return ret;
6060720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
6070720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
6080720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_domain_has_cap(struct iommu_domain *domain,
6090720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko				    unsigned long cap)
6100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
6110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	return 0;
6120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
6130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
6140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void print_ctx_regs(void __iomem *base, int ctx)
6150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
6160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned int fsr = GET_FSR(base, ctx);
6170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	pr_err("FAR    = %08x    PAR    = %08x\n",
6180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	       GET_FAR(base, ctx), GET_PAR(base, ctx));
6190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	pr_err("FSR    = %08x [%s%s%s%s%s%s%s%s%s%s]\n", fsr,
6200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			(fsr & 0x02) ? "TF " : "",
6210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			(fsr & 0x04) ? "AFF " : "",
6220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			(fsr & 0x08) ? "APF " : "",
6230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			(fsr & 0x10) ? "TLBMF " : "",
6240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			(fsr & 0x20) ? "HTWDEEF " : "",
6250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			(fsr & 0x40) ? "HTWSEEF " : "",
6260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			(fsr & 0x80) ? "MHF " : "",
6270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			(fsr & 0x10000) ? "SL " : "",
6280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			(fsr & 0x40000000) ? "SS " : "",
6290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			(fsr & 0x80000000) ? "MULTI " : "");
6300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
6310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	pr_err("FSYNR0 = %08x    FSYNR1 = %08x\n",
6320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	       GET_FSYNR0(base, ctx), GET_FSYNR1(base, ctx));
6330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	pr_err("TTBR0  = %08x    TTBR1  = %08x\n",
6340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	       GET_TTBR0(base, ctx), GET_TTBR1(base, ctx));
6350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	pr_err("SCTLR  = %08x    ACTLR  = %08x\n",
6360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	       GET_SCTLR(base, ctx), GET_ACTLR(base, ctx));
6370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	pr_err("PRRR   = %08x    NMRR   = %08x\n",
6380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	       GET_PRRR(base, ctx), GET_NMRR(base, ctx));
6390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
6400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
6410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkoirqreturn_t msm_iommu_fault_handler(int irq, void *dev_id)
6420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
6430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_drvdata *drvdata = dev_id;
6440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	void __iomem *base;
64533069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko	unsigned int fsr;
646a43d8c101eb71bf4527dd7f36a34a5a502894f38Stepan Moskovchenko	int i, ret;
6470720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
6480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_lock(&msm_iommu_lock);
6490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
6500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (!drvdata) {
6510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		pr_err("Invalid device ID in context interrupt handler\n");
6520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
6530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
6540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
6550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	base = drvdata->base;
6560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
6570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	pr_err("Unexpected IOMMU page fault!\n");
6580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	pr_err("base = %08x\n", (unsigned int) base);
6590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
66041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	ret = __enable_clocks(drvdata);
66141f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	if (ret)
66241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		goto fail;
66341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko
664a43d8c101eb71bf4527dd7f36a34a5a502894f38Stepan Moskovchenko	for (i = 0; i < drvdata->ncb; i++) {
6650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		fsr = GET_FSR(base, i);
6660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		if (fsr) {
6670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			pr_err("Fault occurred in context %d.\n", i);
6680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			pr_err("Interesting registers:\n");
6690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			print_ctx_regs(base, i);
6700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			SET_FSR(base, i, 0x4000000F);
6710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		}
6720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
67341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	__disable_clocks(drvdata);
6740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail:
6750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_unlock(&msm_iommu_lock);
6760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	return 0;
6770720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
6780720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
6790720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic struct iommu_ops msm_iommu_ops = {
6800720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	.domain_init = msm_iommu_domain_init,
6810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	.domain_destroy = msm_iommu_domain_destroy,
6820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	.attach_dev = msm_iommu_attach_dev,
6830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	.detach_dev = msm_iommu_detach_dev,
6840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	.map = msm_iommu_map,
6850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	.unmap = msm_iommu_unmap,
6860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	.iova_to_phys = msm_iommu_iova_to_phys,
6870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	.domain_has_cap = msm_iommu_domain_has_cap
6880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko};
6890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
690100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenkostatic int __init get_tex_class(int icp, int ocp, int mt, int nos)
691100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko{
692100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	int i = 0;
693100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	unsigned int prrr = 0;
694100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	unsigned int nmrr = 0;
695100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	int c_icp, c_ocp, c_mt, c_nos;
696100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
697100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	RCP15_PRRR(prrr);
698100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	RCP15_NMRR(nmrr);
699100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
700100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	for (i = 0; i < NUM_TEX_CLASS; i++) {
701100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		c_nos = PRRR_NOS(prrr, i);
702100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		c_mt = PRRR_MT(prrr, i);
703100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		c_icp = NMRR_ICP(nmrr, i);
704100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		c_ocp = NMRR_OCP(nmrr, i);
705100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
706100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		if (icp == c_icp && ocp == c_ocp && c_mt == mt && c_nos == nos)
707100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko			return i;
708100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	}
709100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
710100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	return -ENODEV;
711100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko}
712100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
713100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenkostatic void __init setup_iommu_tex_classes(void)
714100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko{
715100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	msm_iommu_tex_class[MSM_IOMMU_ATTR_NONCACHED] =
716100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko			get_tex_class(CP_NONCACHED, CP_NONCACHED, MT_NORMAL, 1);
717100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
718100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WB_WA] =
719100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko			get_tex_class(CP_WB_WA, CP_WB_WA, MT_NORMAL, 1);
720100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
721100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WB_NWA] =
722100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko			get_tex_class(CP_WB_NWA, CP_WB_NWA, MT_NORMAL, 1);
723100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
724100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WT] =
725100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko			get_tex_class(CP_WT, CP_WT, MT_NORMAL, 1);
726100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko}
727100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
728516cbc793eb4be5123289d067b54dfcdabeddb25Stepan Moskovchenkostatic int __init msm_iommu_init(void)
7290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
730100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	setup_iommu_tex_classes();
7310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	register_iommu(&msm_iommu_ops);
7320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	return 0;
7330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
7340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
7350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkosubsys_initcall(msm_iommu_init);
7360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
7370720d1f052dc1576396a39b327da6e60082c4efaStepan MoskovchenkoMODULE_LICENSE("GPL v2");
7380720d1f052dc1576396a39b327da6e60082c4efaStepan MoskovchenkoMODULE_AUTHOR("Stepan Moskovchenko <stepanm@codeaurora.org>");
739