141f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved.
20720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *
30720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * This program is free software; you can redistribute it and/or modify
40720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * it under the terms of the GNU General Public License version 2 and
50720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * only version 2 as published by the Free Software Foundation.
60720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *
70720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * This program is distributed in the hope that it will be useful,
80720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * but WITHOUT ANY WARRANTY; without even the implied warranty of
90720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * GNU General Public License for more details.
110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko *
120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * You should have received a copy of the GNU General Public License
130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * along with this program; if not, write to the Free Software
140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko * 02110-1301, USA.
160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko */
170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/kernel.h>
200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/module.h>
210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/platform_device.h>
220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/errno.h>
230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/io.h>
240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/interrupt.h>
250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/list.h>
260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/spinlock.h>
270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/slab.h>
280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <linux/iommu.h>
2941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko#include <linux/clk.h>
300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <asm/cacheflush.h>
320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <asm/sizes.h>
330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <mach/iommu_hw-8xxx.h>
350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#include <mach/iommu.h>
360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
37100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko#define MRC(reg, processor, op1, crn, crm, op2)				\
38100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko__asm__ __volatile__ (							\
39100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko"   mrc   "   #processor "," #op1 ", %0,"  #crn "," #crm "," #op2 "\n"  \
40100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko: "=r" (reg))
41100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
42100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko#define RCP15_PRRR(reg)		MRC(reg, p15, 0, c10, c2, 0)
43100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko#define RCP15_NMRR(reg)		MRC(reg, p15, 0, c10, c2, 1)
44100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
4583427275546a6e36076d4f1a0545335b1bb2afc2Ohad Ben-Cohen/* bitmap of the page sizes currently supported */
4683427275546a6e36076d4f1a0545335b1bb2afc2Ohad Ben-Cohen#define MSM_IOMMU_PGSIZES	(SZ_4K | SZ_64K | SZ_1M | SZ_16M)
4783427275546a6e36076d4f1a0545335b1bb2afc2Ohad Ben-Cohen
48100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenkostatic int msm_iommu_tex_class[4];
49100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
500720d1f052dc1576396a39b327da6e60082c4efaStepan MoskovchenkoDEFINE_SPINLOCK(msm_iommu_lock);
510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostruct msm_priv {
530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long *pgtable;
540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct list_head list_attached;
550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko};
560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenkostatic int __enable_clocks(struct msm_iommu_drvdata *drvdata)
5841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko{
5941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	int ret;
6041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko
6141f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	ret = clk_enable(drvdata->pclk);
6241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	if (ret)
6341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		goto fail;
6441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko
6541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	if (drvdata->clk) {
6641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		ret = clk_enable(drvdata->clk);
6741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		if (ret)
6841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko			clk_disable(drvdata->pclk);
6941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	}
7041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenkofail:
7141f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	return ret;
7241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko}
7341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko
7441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenkostatic void __disable_clocks(struct msm_iommu_drvdata *drvdata)
7541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko{
7641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	if (drvdata->clk)
7741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		clk_disable(drvdata->clk);
7841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	clk_disable(drvdata->pclk);
7941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko}
8041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko
8133069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenkostatic int __flush_iotlb(struct iommu_domain *domain)
820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_priv *priv = domain->priv;
840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_drvdata *iommu_drvdata;
850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_ctx_drvdata *ctx_drvdata;
8633069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko	int ret = 0;
870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#ifndef CONFIG_IOMMU_PGTABLES_L2
880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long *fl_table = priv->pgtable;
890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	int i;
900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
91f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko	if (!list_empty(&priv->list_attached)) {
92f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko		dmac_flush_range(fl_table, fl_table + SZ_16K);
930720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
94f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko		for (i = 0; i < NUM_FL_PTE; i++)
95f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko			if ((fl_table[i] & 0x03) == FL_TYPE_TABLE) {
96f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko				void *sl_table = __va(fl_table[i] &
97f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko								FL_BASE_MASK);
98f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko				dmac_flush_range(sl_table, sl_table + SZ_4K);
99f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko			}
100f6f41eb9ccc0e6fad0ccba4c5e0a97de935db734Stepan Moskovchenko	}
1010720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#endif
1020720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1030720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	list_for_each_entry(ctx_drvdata, &priv->list_attached, attached_elm) {
1040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		if (!ctx_drvdata->pdev || !ctx_drvdata->pdev->dev.parent)
1050720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			BUG();
1060720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1070720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent);
10841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		BUG_ON(!iommu_drvdata);
10941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko
11041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		ret = __enable_clocks(iommu_drvdata);
11141f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		if (ret)
11241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko			goto fail;
11341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko
1140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		SET_CTX_TLBIALL(iommu_drvdata->base, ctx_drvdata->num, 0);
11541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		__disable_clocks(iommu_drvdata);
1160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
11741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenkofail:
11833069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko	return ret;
1190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
1200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void __reset_context(void __iomem *base, int ctx)
1220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
1230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_BPRCOSH(base, ctx, 0);
1240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_BPRCISH(base, ctx, 0);
1250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_BPRCNSH(base, ctx, 0);
1260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_BPSHCFG(base, ctx, 0);
1270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_BPMTCFG(base, ctx, 0);
1280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_ACTLR(base, ctx, 0);
1290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_SCTLR(base, ctx, 0);
1300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_FSRRESTORE(base, ctx, 0);
1310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR0(base, ctx, 0);
1320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR1(base, ctx, 0);
1330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBCR(base, ctx, 0);
1340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_BFBCR(base, ctx, 0);
1350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_PAR(base, ctx, 0);
1360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_FAR(base, ctx, 0);
1370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_CTX_TLBIALL(base, ctx, 0);
1380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TLBFLPTER(base, ctx, 0);
1390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TLBSLPTER(base, ctx, 0);
1400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TLBLKCR(base, ctx, 0);
1410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_PRRR(base, ctx, 0);
1420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_NMRR(base, ctx, 0);
1430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
1440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1450720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void __program_context(void __iomem *base, int ctx, phys_addr_t pgtable)
1460720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
147100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	unsigned int prrr, nmrr;
1480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	__reset_context(base, ctx);
1490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Set up HTW mode */
1510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* TLB miss configuration: perform HTW on miss */
1520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TLBMCFG(base, ctx, 0x3);
1530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* V2P configuration: HTW for access */
1550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_V2PCFG(base, ctx, 0x3);
1560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBCR(base, ctx, 0);
1580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR0_PA(base, ctx, (pgtable >> 14));
1590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Invalidate the TLB for this context */
1610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_CTX_TLBIALL(base, ctx, 0);
1620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Set interrupt number to "secure" interrupt */
1640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_IRPTNDX(base, ctx, 0);
1650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Enable context fault interrupt */
1670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_CFEIE(base, ctx, 1);
1680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Stall access on a context fault and let the handler deal with it */
1700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_CFCFG(base, ctx, 1);
1710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Redirect all cacheable requests to L2 slave port. */
1730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_RCISH(base, ctx, 1);
1740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_RCOSH(base, ctx, 1);
1750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_RCNSH(base, ctx, 1);
1760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1770720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Turn on TEX Remap */
1780720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TRE(base, ctx, 1);
1790720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
180100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	/* Set TEX remap attributes */
181100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	RCP15_PRRR(prrr);
182100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	RCP15_NMRR(nmrr);
183100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	SET_PRRR(base, ctx, prrr);
184100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	SET_NMRR(base, ctx, nmrr);
1850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Turn on BFB prefetch */
1870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_BFBDFE(base, ctx, 1);
1880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#ifdef CONFIG_IOMMU_PGTABLES_L2
1900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Configure page tables as inner-cacheable and shareable to reduce
1910720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	 * the TLB miss penalty.
1920720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	 */
1930720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR0_SH(base, ctx, 1);
1940720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR1_SH(base, ctx, 1);
1950720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR0_NOS(base, ctx, 1);
1970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR1_NOS(base, ctx, 1);
1980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
1990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR0_IRGNH(base, ctx, 0); /* WB, WA */
2000720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR0_IRGNL(base, ctx, 1);
2010720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2020720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR1_IRGNH(base, ctx, 0); /* WB, WA */
2030720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR1_IRGNL(base, ctx, 1);
2040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2050720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR0_ORGN(base, ctx, 1); /* WB, WA */
2060720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_TTBR1_ORGN(base, ctx, 1); /* WB, WA */
2070720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko#endif
2080720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2090720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Enable the MMU */
2100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_M(base, ctx, 1);
2110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
2120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_domain_init(struct iommu_domain *domain)
2140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
2150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_priv *priv = kzalloc(sizeof(*priv), GFP_KERNEL);
2160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (!priv)
2180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail_nomem;
2190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	INIT_LIST_HEAD(&priv->list_attached);
2210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	priv->pgtable = (unsigned long *)__get_free_pages(GFP_KERNEL,
2220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko							  get_order(SZ_16K));
2230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (!priv->pgtable)
2250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail_nomem;
2260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	memset(priv->pgtable, 0, SZ_16K);
2280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	domain->priv = priv;
2290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	return 0;
2300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail_nomem:
2320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	kfree(priv);
2330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	return -ENOMEM;
2340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
2350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void msm_iommu_domain_destroy(struct iommu_domain *domain)
2370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
2380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_priv *priv;
2390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long flags;
2400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long *fl_table;
2410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	int i;
2420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_lock_irqsave(&msm_iommu_lock, flags);
2440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	priv = domain->priv;
2450720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	domain->priv = NULL;
2460720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2470720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (priv) {
2480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		fl_table = priv->pgtable;
2490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		for (i = 0; i < NUM_FL_PTE; i++)
2510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			if ((fl_table[i] & 0x03) == FL_TYPE_TABLE)
2520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko				free_page((unsigned long) __va(((fl_table[i]) &
2530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko								FL_BASE_MASK)));
2540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		free_pages((unsigned long)priv->pgtable, get_order(SZ_16K));
2560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		priv->pgtable = NULL;
2570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
2580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	kfree(priv);
2600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_unlock_irqrestore(&msm_iommu_lock, flags);
2610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
2620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
2640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
2650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_priv *priv;
2660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_ctx_dev *ctx_dev;
2670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_drvdata *iommu_drvdata;
2680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_ctx_drvdata *ctx_drvdata;
2690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_ctx_drvdata *tmp_drvdata;
2700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	int ret = 0;
2710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long flags;
2720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_lock_irqsave(&msm_iommu_lock, flags);
2740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	priv = domain->priv;
2760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2770720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (!priv || !dev) {
2780720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		ret = -EINVAL;
2790720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
2800720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
2810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	iommu_drvdata = dev_get_drvdata(dev->parent);
2830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	ctx_drvdata = dev_get_drvdata(dev);
2840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	ctx_dev = dev->platform_data;
2850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
2860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (!iommu_drvdata || !ctx_drvdata || !ctx_dev) {
2870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		ret = -EINVAL;
2880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
2890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
2900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
29100d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko	if (!list_empty(&ctx_drvdata->attached_elm)) {
29200d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko		ret = -EBUSY;
29300d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko		goto fail;
29400d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko	}
29500d4b2bb03b9ed27f28336b0a68f2bd7f67caa40Stepan Moskovchenko
2960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	list_for_each_entry(tmp_drvdata, &priv->list_attached, attached_elm)
2970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		if (tmp_drvdata == ctx_drvdata) {
2980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			ret = -EBUSY;
2990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			goto fail;
3000720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		}
3010720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
30241f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	ret = __enable_clocks(iommu_drvdata);
30341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	if (ret)
30441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		goto fail;
30541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko
3060720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	__program_context(iommu_drvdata->base, ctx_dev->num,
3070720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			  __pa(priv->pgtable));
3080720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
30941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	__disable_clocks(iommu_drvdata);
3100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	list_add(&(ctx_drvdata->attached_elm), &priv->list_attached);
31133069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko	ret = __flush_iotlb(domain);
3120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail:
3140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_unlock_irqrestore(&msm_iommu_lock, flags);
3150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	return ret;
3160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
3170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void msm_iommu_detach_dev(struct iommu_domain *domain,
3190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko				 struct device *dev)
3200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
3210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_priv *priv;
3220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_ctx_dev *ctx_dev;
3230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_drvdata *iommu_drvdata;
3240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_ctx_drvdata *ctx_drvdata;
3250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long flags;
32633069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko	int ret;
3270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_lock_irqsave(&msm_iommu_lock, flags);
3290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	priv = domain->priv;
3300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (!priv || !dev)
3320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
3330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	iommu_drvdata = dev_get_drvdata(dev->parent);
3350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	ctx_drvdata = dev_get_drvdata(dev);
3360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	ctx_dev = dev->platform_data;
3370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (!iommu_drvdata || !ctx_drvdata || !ctx_dev)
3390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
3400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
34133069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko	ret = __flush_iotlb(domain);
34233069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko	if (ret)
34333069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko		goto fail;
34433069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko
34541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	ret = __enable_clocks(iommu_drvdata);
34641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	if (ret)
34741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		goto fail;
34841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko
3490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	__reset_context(iommu_drvdata->base, ctx_dev->num);
35041f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	__disable_clocks(iommu_drvdata);
3510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	list_del_init(&ctx_drvdata->attached_elm);
3520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail:
3540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_unlock_irqrestore(&msm_iommu_lock, flags);
3550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
3560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_map(struct iommu_domain *domain, unsigned long va,
3585009065d38c95455bd2d27c2838313e3dd0c5bc7Ohad Ben-Cohen			 phys_addr_t pa, size_t len, int prot)
3590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
3600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_priv *priv;
3610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long flags;
3620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long *fl_table;
3630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long *fl_pte;
3640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long fl_offset;
3650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long *sl_table;
3660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long *sl_pte;
3670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long sl_offset;
368100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	unsigned int pgprot;
369100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	int ret = 0, tex, sh;
3700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_lock_irqsave(&msm_iommu_lock, flags);
3720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
373100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	sh = (prot & MSM_IOMMU_ATTR_SH) ? 1 : 0;
374100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	tex = msm_iommu_tex_class[prot & MSM_IOMMU_CP_MASK];
375100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
376100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	if (tex < 0 || tex > NUM_TEX_CLASS - 1) {
377100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		ret = -EINVAL;
378100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		goto fail;
379100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	}
380100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
381100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	priv = domain->priv;
3820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (!priv) {
3830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		ret = -EINVAL;
3840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
3850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
3860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	fl_table = priv->pgtable;
3880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (len != SZ_16M && len != SZ_1M &&
3900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	    len != SZ_64K && len != SZ_4K) {
3910720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		pr_debug("Bad size: %d\n", len);
3920720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		ret = -EINVAL;
3930720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
3940720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
3950720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
3960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (!fl_table) {
3970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		pr_debug("Null page table\n");
3980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		ret = -EINVAL;
3990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
4000720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
4010720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
402100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	if (len == SZ_16M || len == SZ_1M) {
403100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		pgprot = sh ? FL_SHARED : 0;
404100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		pgprot |= tex & 0x01 ? FL_BUFFERABLE : 0;
405100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		pgprot |= tex & 0x02 ? FL_CACHEABLE : 0;
406100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		pgprot |= tex & 0x04 ? FL_TEX0 : 0;
407100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	} else	{
408100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		pgprot = sh ? SL_SHARED : 0;
409100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		pgprot |= tex & 0x01 ? SL_BUFFERABLE : 0;
410100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		pgprot |= tex & 0x02 ? SL_CACHEABLE : 0;
411100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		pgprot |= tex & 0x04 ? SL_TEX0 : 0;
412100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	}
413100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
4140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	fl_offset = FL_OFFSET(va);	/* Upper 12 bits */
4150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	fl_pte = fl_table + fl_offset;	/* int pointers, 4 bytes */
4160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (len == SZ_16M) {
4180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		int i = 0;
4190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		for (i = 0; i < 16; i++)
4200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			*(fl_pte+i) = (pa & 0xFF000000) | FL_SUPERSECTION |
4210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko				  FL_AP_READ | FL_AP_WRITE | FL_TYPE_SECT |
4222e8c8ba98376459e73d03a285f5d3406b630ea2dStepan Moskovchenko				  FL_SHARED | FL_NG | pgprot;
4230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
4240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (len == SZ_1M)
4262e8c8ba98376459e73d03a285f5d3406b630ea2dStepan Moskovchenko		*fl_pte = (pa & 0xFFF00000) | FL_AP_READ | FL_AP_WRITE | FL_NG |
427100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko					    FL_TYPE_SECT | FL_SHARED | pgprot;
4280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Need a 2nd level table */
4300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if ((len == SZ_4K || len == SZ_64K) && (*fl_pte) == 0) {
4310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		unsigned long *sl;
432294b2dea83ba0a6d6034a7521bc62c317efab17bStepan Moskovchenko		sl = (unsigned long *) __get_free_pages(GFP_ATOMIC,
4330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko							get_order(SZ_4K));
4340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		if (!sl) {
4360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			pr_debug("Could not allocate second level table\n");
4370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			ret = -ENOMEM;
4380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			goto fail;
4390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		}
4400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		memset(sl, 0, SZ_4K);
4420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		*fl_pte = ((((int)__pa(sl)) & FL_BASE_MASK) | FL_TYPE_TABLE);
4430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
4440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4450720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK));
4460720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	sl_offset = SL_OFFSET(va);
4470720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	sl_pte = sl_table + sl_offset;
4480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (len == SZ_4K)
4512e8c8ba98376459e73d03a285f5d3406b630ea2dStepan Moskovchenko		*sl_pte = (pa & SL_BASE_MASK_SMALL) | SL_AP0 | SL_AP1 | SL_NG |
452100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko					  SL_SHARED | SL_TYPE_SMALL | pgprot;
4530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (len == SZ_64K) {
4550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		int i;
4560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		for (i = 0; i < 16; i++)
4580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			*(sl_pte+i) = (pa & SL_BASE_MASK_LARGE) | SL_AP0 |
4592e8c8ba98376459e73d03a285f5d3406b630ea2dStepan Moskovchenko			    SL_NG | SL_AP1 | SL_SHARED | SL_TYPE_LARGE | pgprot;
4600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
4610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
46233069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko	ret = __flush_iotlb(domain);
4630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail:
4640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_unlock_irqrestore(&msm_iommu_lock, flags);
4650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	return ret;
4660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
4670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4685009065d38c95455bd2d27c2838313e3dd0c5bc7Ohad Ben-Cohenstatic size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long va,
4695009065d38c95455bd2d27c2838313e3dd0c5bc7Ohad Ben-Cohen			    size_t len)
4700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
4710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_priv *priv;
4720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long flags;
4730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long *fl_table;
4740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long *fl_pte;
4750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long fl_offset;
4760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long *sl_table;
4770720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long *sl_pte;
4780720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long sl_offset;
4790720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	int i, ret = 0;
4800720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_lock_irqsave(&msm_iommu_lock, flags);
4820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	priv = domain->priv;
4840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
48505df1f3c2afaef5672627f2b7095f0d4c4dbc3a0Joerg Roedel	if (!priv)
4860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
4870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	fl_table = priv->pgtable;
4890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (len != SZ_16M && len != SZ_1M &&
4910720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	    len != SZ_64K && len != SZ_4K) {
4920720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		pr_debug("Bad length: %d\n", len);
4930720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
4940720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
4950720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
4960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (!fl_table) {
4970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		pr_debug("Null page table\n");
4980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
4990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
5000720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5010720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	fl_offset = FL_OFFSET(va);	/* Upper 12 bits */
5020720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	fl_pte = fl_table + fl_offset;	/* int pointers, 4 bytes */
5030720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (*fl_pte == 0) {
5050720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		pr_debug("First level PTE is 0\n");
5060720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
5070720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
5080720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5090720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Unmap supersection */
5100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (len == SZ_16M)
5110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		for (i = 0; i < 16; i++)
5120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			*(fl_pte+i) = 0;
5130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (len == SZ_1M)
5150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		*fl_pte = 0;
5160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK));
5180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	sl_offset = SL_OFFSET(va);
5190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	sl_pte = sl_table + sl_offset;
5200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (len == SZ_64K) {
5220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		for (i = 0; i < 16; i++)
5230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			*(sl_pte+i) = 0;
5240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
5250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (len == SZ_4K)
5270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		*sl_pte = 0;
5280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (len == SZ_4K || len == SZ_64K) {
5300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		int used = 0;
5310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		for (i = 0; i < NUM_SL_PTE; i++)
5330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			if (sl_table[i])
5340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko				used = 1;
5350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		if (!used) {
5360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			free_page((unsigned long)sl_table);
5370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			*fl_pte = 0;
5380720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		}
5390720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
5400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
54133069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko	ret = __flush_iotlb(domain);
5429e28547f887c3040747d8422e1feef79bea93bd8Ohad Ben-Cohen
5430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail:
5440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_unlock_irqrestore(&msm_iommu_lock, flags);
5455009065d38c95455bd2d27c2838313e3dd0c5bc7Ohad Ben-Cohen
5465009065d38c95455bd2d27c2838313e3dd0c5bc7Ohad Ben-Cohen	/* the IOMMU API requires us to return how many bytes were unmapped */
5475009065d38c95455bd2d27c2838313e3dd0c5bc7Ohad Ben-Cohen	len = ret ? 0 : len;
5485009065d38c95455bd2d27c2838313e3dd0c5bc7Ohad Ben-Cohen	return len;
5490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
5500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain,
5520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko					  unsigned long va)
5530720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
5540720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_priv *priv;
5550720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_drvdata *iommu_drvdata;
5560720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_ctx_drvdata *ctx_drvdata;
5570720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned int par;
5580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned long flags;
5590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	void __iomem *base;
5600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	phys_addr_t ret = 0;
5610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	int ctx;
5620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_lock_irqsave(&msm_iommu_lock, flags);
5640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	priv = domain->priv;
5660720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (list_empty(&priv->list_attached))
5670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
5680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	ctx_drvdata = list_entry(priv->list_attached.next,
5700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko				 struct msm_iommu_ctx_drvdata, attached_elm);
5710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent);
5720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	base = iommu_drvdata->base;
5740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	ctx = ctx_drvdata->num;
5750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
57641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	ret = __enable_clocks(iommu_drvdata);
57741f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	if (ret)
57841f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		goto fail;
57941f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko
5800720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* Invalidate context TLB */
5810720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	SET_CTX_TLBIALL(base, ctx, 0);
582b0e7808d548ea1d857216d31d63078411203a116Stepan Moskovchenko	SET_V2PPR(base, ctx, va & V2Pxx_VA);
5830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5840720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	par = GET_PAR(base, ctx);
5850720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
5860720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	/* We are dealing with a supersection */
5870720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (GET_NOFAULT_SS(base, ctx))
5880720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		ret = (par & 0xFF000000) | (va & 0x00FFFFFF);
5890720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	else	/* Upper 20 bits from PAR, lower 12 from VA */
5900720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		ret = (par & 0xFFFFF000) | (va & 0x00000FFF);
5910720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
59233069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko	if (GET_FAULT(base, ctx))
59333069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko		ret = 0;
59433069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko
59541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	__disable_clocks(iommu_drvdata);
5960720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail:
5970720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_unlock_irqrestore(&msm_iommu_lock, flags);
5980720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	return ret;
5990720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
6000720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
6010720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic int msm_iommu_domain_has_cap(struct iommu_domain *domain,
6020720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko				    unsigned long cap)
6030720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
6040720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	return 0;
6050720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
6060720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
6070720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic void print_ctx_regs(void __iomem *base, int ctx)
6080720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
6090720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	unsigned int fsr = GET_FSR(base, ctx);
6100720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	pr_err("FAR    = %08x    PAR    = %08x\n",
6110720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	       GET_FAR(base, ctx), GET_PAR(base, ctx));
6120720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	pr_err("FSR    = %08x [%s%s%s%s%s%s%s%s%s%s]\n", fsr,
6130720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			(fsr & 0x02) ? "TF " : "",
6140720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			(fsr & 0x04) ? "AFF " : "",
6150720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			(fsr & 0x08) ? "APF " : "",
6160720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			(fsr & 0x10) ? "TLBMF " : "",
6170720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			(fsr & 0x20) ? "HTWDEEF " : "",
6180720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			(fsr & 0x40) ? "HTWSEEF " : "",
6190720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			(fsr & 0x80) ? "MHF " : "",
6200720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			(fsr & 0x10000) ? "SL " : "",
6210720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			(fsr & 0x40000000) ? "SS " : "",
6220720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			(fsr & 0x80000000) ? "MULTI " : "");
6230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
6240720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	pr_err("FSYNR0 = %08x    FSYNR1 = %08x\n",
6250720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	       GET_FSYNR0(base, ctx), GET_FSYNR1(base, ctx));
6260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	pr_err("TTBR0  = %08x    TTBR1  = %08x\n",
6270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	       GET_TTBR0(base, ctx), GET_TTBR1(base, ctx));
6280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	pr_err("SCTLR  = %08x    ACTLR  = %08x\n",
6290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	       GET_SCTLR(base, ctx), GET_ACTLR(base, ctx));
6300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	pr_err("PRRR   = %08x    NMRR   = %08x\n",
6310720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	       GET_PRRR(base, ctx), GET_NMRR(base, ctx));
6320720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
6330720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
6340720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkoirqreturn_t msm_iommu_fault_handler(int irq, void *dev_id)
6350720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
6360720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	struct msm_iommu_drvdata *drvdata = dev_id;
6370720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	void __iomem *base;
63833069739d1c2f03734d5b5b003593e9552d412c1Stepan Moskovchenko	unsigned int fsr;
639a43d8c101eb71bf4527dd7f36a34a5a502894f38Stepan Moskovchenko	int i, ret;
6400720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
6410720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_lock(&msm_iommu_lock);
6420720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
6430720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	if (!drvdata) {
6440720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		pr_err("Invalid device ID in context interrupt handler\n");
6450720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		goto fail;
6460720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
6470720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
6480720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	base = drvdata->base;
6490720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
6500720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	pr_err("Unexpected IOMMU page fault!\n");
6510720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	pr_err("base = %08x\n", (unsigned int) base);
6520720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
65341f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	ret = __enable_clocks(drvdata);
65441f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	if (ret)
65541f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko		goto fail;
65641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko
657a43d8c101eb71bf4527dd7f36a34a5a502894f38Stepan Moskovchenko	for (i = 0; i < drvdata->ncb; i++) {
6580720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		fsr = GET_FSR(base, i);
6590720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		if (fsr) {
6600720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			pr_err("Fault occurred in context %d.\n", i);
6610720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			pr_err("Interesting registers:\n");
6620720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			print_ctx_regs(base, i);
6630720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko			SET_FSR(base, i, 0x4000000F);
6640720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko		}
6650720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	}
66641f3f5138a5ea71ee603f3d556953fce9aba3074Stepan Moskovchenko	__disable_clocks(drvdata);
6670720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkofail:
6680720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	spin_unlock(&msm_iommu_lock);
6690720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	return 0;
6700720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
6710720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
6720720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkostatic struct iommu_ops msm_iommu_ops = {
6730720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	.domain_init = msm_iommu_domain_init,
6740720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	.domain_destroy = msm_iommu_domain_destroy,
6750720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	.attach_dev = msm_iommu_attach_dev,
6760720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	.detach_dev = msm_iommu_detach_dev,
6770720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	.map = msm_iommu_map,
6780720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	.unmap = msm_iommu_unmap,
6790720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	.iova_to_phys = msm_iommu_iova_to_phys,
68083427275546a6e36076d4f1a0545335b1bb2afc2Ohad Ben-Cohen	.domain_has_cap = msm_iommu_domain_has_cap,
68183427275546a6e36076d4f1a0545335b1bb2afc2Ohad Ben-Cohen	.pgsize_bitmap = MSM_IOMMU_PGSIZES,
6820720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko};
6830720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
684100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenkostatic int __init get_tex_class(int icp, int ocp, int mt, int nos)
685100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko{
686100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	int i = 0;
687100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	unsigned int prrr = 0;
688100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	unsigned int nmrr = 0;
689100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	int c_icp, c_ocp, c_mt, c_nos;
690100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
691100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	RCP15_PRRR(prrr);
692100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	RCP15_NMRR(nmrr);
693100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
694100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	for (i = 0; i < NUM_TEX_CLASS; i++) {
695100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		c_nos = PRRR_NOS(prrr, i);
696100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		c_mt = PRRR_MT(prrr, i);
697100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		c_icp = NMRR_ICP(nmrr, i);
698100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		c_ocp = NMRR_OCP(nmrr, i);
699100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
700100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko		if (icp == c_icp && ocp == c_ocp && c_mt == mt && c_nos == nos)
701100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko			return i;
702100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	}
703100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
704100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	return -ENODEV;
705100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko}
706100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
707100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenkostatic void __init setup_iommu_tex_classes(void)
708100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko{
709100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	msm_iommu_tex_class[MSM_IOMMU_ATTR_NONCACHED] =
710100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko			get_tex_class(CP_NONCACHED, CP_NONCACHED, MT_NORMAL, 1);
711100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
712100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WB_WA] =
713100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko			get_tex_class(CP_WB_WA, CP_WB_WA, MT_NORMAL, 1);
714100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
715100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WB_NWA] =
716100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko			get_tex_class(CP_WB_NWA, CP_WB_NWA, MT_NORMAL, 1);
717100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
718100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WT] =
719100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko			get_tex_class(CP_WT, CP_WT, MT_NORMAL, 1);
720100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko}
721100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko
722516cbc793eb4be5123289d067b54dfcdabeddb25Stepan Moskovchenkostatic int __init msm_iommu_init(void)
7230720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko{
724100832c9b6adb3d63407416931caeba3f3b9a777Stepan Moskovchenko	setup_iommu_tex_classes();
72585eebbc5e4633e6eee10bdddd00d175daadd5841Joerg Roedel	bus_set_iommu(&platform_bus_type, &msm_iommu_ops);
7260720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko	return 0;
7270720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko}
7280720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
7290720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenkosubsys_initcall(msm_iommu_init);
7300720d1f052dc1576396a39b327da6e60082c4efaStepan Moskovchenko
7310720d1f052dc1576396a39b327da6e60082c4efaStepan MoskovchenkoMODULE_LICENSE("GPL v2");
7320720d1f052dc1576396a39b327da6e60082c4efaStepan MoskovchenkoMODULE_AUTHOR("Stepan Moskovchenko <stepanm@codeaurora.org>");
733