1173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/*
2173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * Freescale MPC85xx, MPC83xx DMA Engine support
3173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei *
4e2c8e425baa01a4c8e6ae1b90194ed3d3cde0c66Li Yang * Copyright (C) 2007-2010 Freescale Semiconductor, Inc. All rights reserved.
5173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei *
6173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * Author:
7173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei *   Zhang Wei <wei.zhang@freescale.com>, Jul 2007
8173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei *   Ebony Zhu <ebony.zhu@freescale.com>, May 2007
9173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei *
10173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * Description:
11173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei *   DMA engine driver for Freescale MPC8540 DMA controller, which is
12173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei *   also fit for MPC8560, MPC8555, MPC8548, MPC8641, and etc.
13c2e07b3a9ced33dd92597201be3931be8ea57ed6Stefan Weil *   The support for MPC8349 DMA controller is also added.
14173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei *
15a7aea373b4ca428f1be2c1fedd2f26c8e3f2864dIra W. Snyder * This driver instructs the DMA controller to issue the PCI Read Multiple
16a7aea373b4ca428f1be2c1fedd2f26c8e3f2864dIra W. Snyder * command for PCI read operations, instead of using the default PCI Read Line
17a7aea373b4ca428f1be2c1fedd2f26c8e3f2864dIra W. Snyder * command. Please be aware that this setting may result in read pre-fetching
18a7aea373b4ca428f1be2c1fedd2f26c8e3f2864dIra W. Snyder * on some platforms.
19a7aea373b4ca428f1be2c1fedd2f26c8e3f2864dIra W. Snyder *
20173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * This is free software; you can redistribute it and/or modify
21173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * it under the terms of the GNU General Public License as published by
22173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * the Free Software Foundation; either version 2 of the License, or
23173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * (at your option) any later version.
24173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei *
25173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */
26173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
27173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei#include <linux/init.h>
28173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei#include <linux/module.h>
29173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei#include <linux/pci.h>
305a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h>
31173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei#include <linux/interrupt.h>
32173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei#include <linux/dmaengine.h>
33173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei#include <linux/delay.h>
34173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei#include <linux/dma-mapping.h>
35173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei#include <linux/dmapool.h>
36173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei#include <linux/of_platform.h>
37173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
38d2ebfb335b0426deb1a4fb14e4e926d81ecd8235Russell King - ARM Linux#include "dmaengine.h"
39173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei#include "fsldma.h"
40173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
41b158471ef63bf399165db96e945a828096502d9dIra Snyder#define chan_dbg(chan, fmt, arg...)					\
42b158471ef63bf399165db96e945a828096502d9dIra Snyder	dev_dbg(chan->dev, "%s: " fmt, chan->name, ##arg)
43b158471ef63bf399165db96e945a828096502d9dIra Snyder#define chan_err(chan, fmt, arg...)					\
44b158471ef63bf399165db96e945a828096502d9dIra Snyder	dev_err(chan->dev, "%s: " fmt, chan->name, ##arg)
45c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
46b158471ef63bf399165db96e945a828096502d9dIra Snyderstatic const char msg_ld_oom[] = "No free memory for link descriptor";
47173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
48e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder/*
49e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder * Register Helpers
50e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder */
51173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
52a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void set_sr(struct fsldma_chan *chan, u32 val)
53173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
54a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	DMA_OUT(chan, &chan->regs->sr, val, 32);
55173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
56173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
57a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic u32 get_sr(struct fsldma_chan *chan)
58173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
59a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	return DMA_IN(chan, &chan->regs->sr, 32);
60173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
61173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
62e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyderstatic void set_cdar(struct fsldma_chan *chan, dma_addr_t addr)
63e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder{
64e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder	DMA_OUT(chan, &chan->regs->cdar, addr | FSL_DMA_SNEN, 64);
65e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder}
66e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder
67e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyderstatic dma_addr_t get_cdar(struct fsldma_chan *chan)
68e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder{
69e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder	return DMA_IN(chan, &chan->regs->cdar, 64) & ~FSL_DMA_SNEN;
70e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder}
71e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder
72e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyderstatic u32 get_bcr(struct fsldma_chan *chan)
73e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder{
74e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder	return DMA_IN(chan, &chan->regs->bcr, 32);
75e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder}
76e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder
77e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder/*
78e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder * Descriptor Helpers
79e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder */
80e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder
81a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void set_desc_cnt(struct fsldma_chan *chan,
82173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei				struct fsl_dma_ld_hw *hw, u32 count)
83173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
84a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	hw->count = CPU_TO_DMA(chan, count, 32);
85173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
86173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
879c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyderstatic u32 get_desc_cnt(struct fsldma_chan *chan, struct fsl_desc_sw *desc)
889c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder{
899c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	return DMA_TO_CPU(chan, desc->hw.count, 32);
909c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder}
919c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder
92a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void set_desc_src(struct fsldma_chan *chan,
9331f4306c83a2daa3e348056b720de511bffe5a9bIra Snyder			 struct fsl_dma_ld_hw *hw, dma_addr_t src)
94173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
95173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	u64 snoop_bits;
96173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
97a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX)
98173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		? ((u64)FSL_DMA_SATR_SREADTYPE_SNOOP_READ << 32) : 0;
99a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	hw->src_addr = CPU_TO_DMA(chan, snoop_bits | src, 64);
100173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
101173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1029c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyderstatic dma_addr_t get_desc_src(struct fsldma_chan *chan,
1039c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder			       struct fsl_desc_sw *desc)
1049c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder{
1059c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	u64 snoop_bits;
1069c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder
1079c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX)
1089c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder		? ((u64)FSL_DMA_SATR_SREADTYPE_SNOOP_READ << 32) : 0;
1099c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	return DMA_TO_CPU(chan, desc->hw.src_addr, 64) & ~snoop_bits;
1109c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder}
1119c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder
112a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void set_desc_dst(struct fsldma_chan *chan,
11331f4306c83a2daa3e348056b720de511bffe5a9bIra Snyder			 struct fsl_dma_ld_hw *hw, dma_addr_t dst)
114173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
115173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	u64 snoop_bits;
116173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
117a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX)
118173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		? ((u64)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE << 32) : 0;
119a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	hw->dst_addr = CPU_TO_DMA(chan, snoop_bits | dst, 64);
120173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
121173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1229c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyderstatic dma_addr_t get_desc_dst(struct fsldma_chan *chan,
1239c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder			       struct fsl_desc_sw *desc)
1249c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder{
1259c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	u64 snoop_bits;
1269c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder
1279c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX)
1289c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder		? ((u64)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE << 32) : 0;
1299c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	return DMA_TO_CPU(chan, desc->hw.dst_addr, 64) & ~snoop_bits;
1309c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder}
1319c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder
132a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void set_desc_next(struct fsldma_chan *chan,
13331f4306c83a2daa3e348056b720de511bffe5a9bIra Snyder			  struct fsl_dma_ld_hw *hw, dma_addr_t next)
134173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
135173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	u64 snoop_bits;
136173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
137a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX)
138173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		? FSL_DMA_SNEN : 0;
139a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	hw->next_ln_addr = CPU_TO_DMA(chan, snoop_bits | next, 64);
140173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
141173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
14231f4306c83a2daa3e348056b720de511bffe5a9bIra Snyderstatic void set_ld_eol(struct fsldma_chan *chan, struct fsl_desc_sw *desc)
143173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
144e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder	u64 snoop_bits;
145173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
146e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder	snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX)
147e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder		? FSL_DMA_SNEN : 0;
148173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
149e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder	desc->hw.next_ln_addr = CPU_TO_DMA(chan,
150e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder		DMA_TO_CPU(chan, desc->hw.next_ln_addr, 64) | FSL_DMA_EOL
151e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder			| snoop_bits, 64);
152173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
153173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
154e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder/*
155e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder * DMA Engine Hardware Control Helpers
156e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder */
157e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder
158e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyderstatic void dma_init(struct fsldma_chan *chan)
159f79abb627f033c85a6088231f20c85bc4a9bd757Zhang Wei{
160e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder	/* Reset the channel */
161e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder	DMA_OUT(chan, &chan->regs->mr, 0, 32);
162e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder
163e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder	switch (chan->feature & FSL_DMA_IP_MASK) {
164e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder	case FSL_DMA_IP_85XX:
165e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder		/* Set the channel to below modes:
166e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder		 * EIE - Error interrupt enable
167e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder		 * EOLNIE - End of links interrupt enable
168e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder		 * BWC - Bandwidth sharing among channels
169e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder		 */
170e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder		DMA_OUT(chan, &chan->regs->mr, FSL_DMA_MR_BWC
171f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder				| FSL_DMA_MR_EIE | FSL_DMA_MR_EOLNIE, 32);
172e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder		break;
173e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder	case FSL_DMA_IP_83XX:
174e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder		/* Set the channel to below modes:
175e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder		 * EOTIE - End-of-transfer interrupt enable
176e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder		 * PRC_RM - PCI read multiple
177e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder		 */
178e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder		DMA_OUT(chan, &chan->regs->mr, FSL_DMA_MR_EOTIE
179e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder				| FSL_DMA_MR_PRC_RM, 32);
180e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder		break;
181e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder	}
182f79abb627f033c85a6088231f20c85bc4a9bd757Zhang Wei}
183f79abb627f033c85a6088231f20c85bc4a9bd757Zhang Wei
184a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic int dma_is_idle(struct fsldma_chan *chan)
185173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
186a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	u32 sr = get_sr(chan);
187173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	return (!(sr & FSL_DMA_SR_CB)) || (sr & FSL_DMA_SR_CH);
188173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
189173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
190f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder/*
191f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder * Start the DMA controller
192f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder *
193f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder * Preconditions:
194f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder * - the CDAR register must point to the start descriptor
195f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder * - the MRn[CS] bit must be cleared
196f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder */
197a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void dma_start(struct fsldma_chan *chan)
198173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
199272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder	u32 mode;
200272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder
201a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	mode = DMA_IN(chan, &chan->regs->mr, 32);
202272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder
203f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	if (chan->feature & FSL_DMA_CHAN_PAUSE_EXT) {
204f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder		DMA_OUT(chan, &chan->regs->bcr, 0, 32);
205f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder		mode |= FSL_DMA_MR_EMP_EN;
206f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	} else {
207f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder		mode &= ~FSL_DMA_MR_EMP_EN;
20843a1a3ed6bf5a1b9ae197b4f5f20033baf19db61Ira Snyder	}
209173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
210f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	if (chan->feature & FSL_DMA_CHAN_START_EXT) {
211272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder		mode |= FSL_DMA_MR_EMS_EN;
212f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	} else {
213f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder		mode &= ~FSL_DMA_MR_EMS_EN;
214272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder		mode |= FSL_DMA_MR_CS;
215f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	}
216173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
217a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	DMA_OUT(chan, &chan->regs->mr, mode, 32);
218173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
219173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
220a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void dma_halt(struct fsldma_chan *chan)
221173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
222272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder	u32 mode;
223900325a6ce33995688b7a680a34e7698f16f4d72Dan Williams	int i;
224900325a6ce33995688b7a680a34e7698f16f4d72Dan Williams
225a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder	/* read the mode register */
226a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	mode = DMA_IN(chan, &chan->regs->mr, 32);
227272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder
228a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder	/*
229a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder	 * The 85xx controller supports channel abort, which will stop
230a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder	 * the current transfer. On 83xx, this bit is the transfer error
231a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder	 * mask bit, which should not be changed.
232a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder	 */
233a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder	if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) {
234a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder		mode |= FSL_DMA_MR_CA;
235a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder		DMA_OUT(chan, &chan->regs->mr, mode, 32);
236a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder
237a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder		mode &= ~FSL_DMA_MR_CA;
238a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder	}
239a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder
240a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder	/* stop the DMA controller */
241a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder	mode &= ~(FSL_DMA_MR_CS | FSL_DMA_MR_EMS_EN);
242a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	DMA_OUT(chan, &chan->regs->mr, mode, 32);
243173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
244a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder	/* wait for the DMA controller to become idle */
245900325a6ce33995688b7a680a34e7698f16f4d72Dan Williams	for (i = 0; i < 100; i++) {
246a1c03319018061304be28d131073ac13a5cb86fbIra Snyder		if (dma_is_idle(chan))
2479c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder			return;
2489c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder
249173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		udelay(10);
250900325a6ce33995688b7a680a34e7698f16f4d72Dan Williams	}
251272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder
2529c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	if (!dma_is_idle(chan))
253b158471ef63bf399165db96e945a828096502d9dIra Snyder		chan_err(chan, "DMA halt timeout!\n");
254173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
255173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
256173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/**
257173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * fsl_chan_set_src_loop_size - Set source address hold transfer size
258a1c03319018061304be28d131073ac13a5cb86fbIra Snyder * @chan : Freescale DMA channel
259173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * @size     : Address loop size, 0 for disable loop
260173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei *
261173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * The set source address hold transfer size. The source
262173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * address hold or loop transfer size is when the DMA transfer
263173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * data from source address (SA), if the loop size is 4, the DMA will
264173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * read data from SA, SA + 1, SA + 2, SA + 3, then loop back to SA,
265173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * SA + 1 ... and so on.
266173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */
267a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void fsl_chan_set_src_loop_size(struct fsldma_chan *chan, int size)
268173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
269272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder	u32 mode;
270272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder
271a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	mode = DMA_IN(chan, &chan->regs->mr, 32);
272272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder
273173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	switch (size) {
274173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	case 0:
275272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder		mode &= ~FSL_DMA_MR_SAHE;
276173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		break;
277173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	case 1:
278173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	case 2:
279173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	case 4:
280173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	case 8:
281272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder		mode |= FSL_DMA_MR_SAHE | (__ilog2(size) << 14);
282173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		break;
283173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	}
284272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder
285a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	DMA_OUT(chan, &chan->regs->mr, mode, 32);
286173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
287173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
288173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/**
289738f5f7e1ae876448cb7d9c82bea258b69386647Ira Snyder * fsl_chan_set_dst_loop_size - Set destination address hold transfer size
290a1c03319018061304be28d131073ac13a5cb86fbIra Snyder * @chan : Freescale DMA channel
291173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * @size     : Address loop size, 0 for disable loop
292173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei *
293173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * The set destination address hold transfer size. The destination
294173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * address hold or loop transfer size is when the DMA transfer
295173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * data to destination address (TA), if the loop size is 4, the DMA will
296173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * write data to TA, TA + 1, TA + 2, TA + 3, then loop back to TA,
297173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * TA + 1 ... and so on.
298173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */
299a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void fsl_chan_set_dst_loop_size(struct fsldma_chan *chan, int size)
300173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
301272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder	u32 mode;
302272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder
303a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	mode = DMA_IN(chan, &chan->regs->mr, 32);
304272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder
305173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	switch (size) {
306173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	case 0:
307272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder		mode &= ~FSL_DMA_MR_DAHE;
308173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		break;
309173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	case 1:
310173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	case 2:
311173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	case 4:
312173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	case 8:
313272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder		mode |= FSL_DMA_MR_DAHE | (__ilog2(size) << 16);
314173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		break;
315173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	}
316272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder
317a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	DMA_OUT(chan, &chan->regs->mr, mode, 32);
318173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
319173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
320173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/**
321e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * fsl_chan_set_request_count - Set DMA Request Count for external control
322a1c03319018061304be28d131073ac13a5cb86fbIra Snyder * @chan : Freescale DMA channel
323e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * @size     : Number of bytes to transfer in a single request
324e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder *
325e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * The Freescale DMA channel can be controlled by the external signal DREQ#.
326e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * The DMA request count is how many bytes are allowed to transfer before
327e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * pausing the channel, after which a new assertion of DREQ# resumes channel
328e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * operation.
329173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei *
330e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * A size of 0 disables external pause control. The maximum size is 1024.
331173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */
332a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void fsl_chan_set_request_count(struct fsldma_chan *chan, int size)
333173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
334272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder	u32 mode;
335272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder
336e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder	BUG_ON(size > 1024);
337272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder
338a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	mode = DMA_IN(chan, &chan->regs->mr, 32);
339272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder	mode |= (__ilog2(size) << 24) & 0x0f000000;
340272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder
341a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	DMA_OUT(chan, &chan->regs->mr, mode, 32);
342e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder}
343173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
344e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder/**
345e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * fsl_chan_toggle_ext_pause - Toggle channel external pause status
346a1c03319018061304be28d131073ac13a5cb86fbIra Snyder * @chan : Freescale DMA channel
347e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * @enable   : 0 is disabled, 1 is enabled.
348e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder *
349e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * The Freescale DMA channel can be controlled by the external signal DREQ#.
350e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * The DMA Request Count feature should be used in addition to this feature
351e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * to set the number of bytes to transfer before pausing the channel.
352e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder */
353a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void fsl_chan_toggle_ext_pause(struct fsldma_chan *chan, int enable)
354e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder{
355e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder	if (enable)
356a1c03319018061304be28d131073ac13a5cb86fbIra Snyder		chan->feature |= FSL_DMA_CHAN_PAUSE_EXT;
357e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder	else
358a1c03319018061304be28d131073ac13a5cb86fbIra Snyder		chan->feature &= ~FSL_DMA_CHAN_PAUSE_EXT;
359173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
360173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
361173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/**
362173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * fsl_chan_toggle_ext_start - Toggle channel external start status
363a1c03319018061304be28d131073ac13a5cb86fbIra Snyder * @chan : Freescale DMA channel
364173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * @enable   : 0 is disabled, 1 is enabled.
365173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei *
366173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * If enable the external start, the channel can be started by an
367173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * external DMA start pin. So the dma_start() does not start the
368173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * transfer immediately. The DMA channel will wait for the
369173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * control pin asserted.
370173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */
371a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void fsl_chan_toggle_ext_start(struct fsldma_chan *chan, int enable)
372173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
373173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	if (enable)
374a1c03319018061304be28d131073ac13a5cb86fbIra Snyder		chan->feature |= FSL_DMA_CHAN_START_EXT;
375173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	else
376a1c03319018061304be28d131073ac13a5cb86fbIra Snyder		chan->feature &= ~FSL_DMA_CHAN_START_EXT;
377173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
378173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
37931f4306c83a2daa3e348056b720de511bffe5a9bIra Snyderstatic void append_ld_queue(struct fsldma_chan *chan, struct fsl_desc_sw *desc)
3809c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder{
3819c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	struct fsl_desc_sw *tail = to_fsl_desc(chan->ld_pending.prev);
3829c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder
3839c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	if (list_empty(&chan->ld_pending))
3849c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder		goto out_splice;
3859c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder
3869c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	/*
3879c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 * Add the hardware descriptor to the chain of hardware descriptors
3889c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 * that already exists in memory.
3899c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 *
3909c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 * This will un-set the EOL bit of the existing transaction, and the
3919c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 * last link in this transaction will become the EOL descriptor.
3929c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 */
3939c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	set_desc_next(chan, &tail->hw, desc->async_tx.phys);
3949c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder
3959c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	/*
3969c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 * Add the software descriptor and all children to the list
3979c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 * of pending transactions
3989c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 */
3999c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyderout_splice:
4009c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	list_splice_tail_init(&desc->tx_list, &chan->ld_pending);
4019c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder}
4029c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder
403173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Weistatic dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)
404173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
405a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	struct fsldma_chan *chan = to_fsl_chan(tx->chan);
406eda34234578fd822c950fd06b5c5ff7ac08b3001Dan Williams	struct fsl_desc_sw *desc = tx_to_fsl_desc(tx);
407eda34234578fd822c950fd06b5c5ff7ac08b3001Dan Williams	struct fsl_desc_sw *child;
408173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	unsigned long flags;
409173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	dma_cookie_t cookie;
410173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
411a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	spin_lock_irqsave(&chan->desc_lock, flags);
412173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
4139c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	/*
4149c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 * assign cookies to all of the software descriptors
4159c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 * that make up this transaction
4169c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 */
417eda34234578fd822c950fd06b5c5ff7ac08b3001Dan Williams	list_for_each_entry(child, &desc->tx_list, node) {
418884485e1f12dcd39390f042e772cdbefc9ebb750Russell King - ARM Linux		cookie = dma_cookie_assign(&child->async_tx);
419bcfb7465c03a8c62c89da374677df56f6b894d44Ira Snyder	}
420bcfb7465c03a8c62c89da374677df56f6b894d44Ira Snyder
4219c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	/* put this transaction onto the tail of the pending queue */
422a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	append_ld_queue(chan, desc);
423173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
424a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	spin_unlock_irqrestore(&chan->desc_lock, flags);
425173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
426173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	return cookie;
427173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
428173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
429173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/**
430173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * fsl_dma_alloc_descriptor - Allocate descriptor from channel's DMA pool.
431a1c03319018061304be28d131073ac13a5cb86fbIra Snyder * @chan : Freescale DMA channel
432173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei *
433173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * Return - The descriptor allocated. NULL for failed.
434173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */
43531f4306c83a2daa3e348056b720de511bffe5a9bIra Snyderstatic struct fsl_desc_sw *fsl_dma_alloc_descriptor(struct fsldma_chan *chan)
436173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
4379c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	struct fsl_desc_sw *desc;
438173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	dma_addr_t pdesc;
4399c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder
4409c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	desc = dma_pool_alloc(chan->desc_pool, GFP_ATOMIC, &pdesc);
4419c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	if (!desc) {
442b158471ef63bf399165db96e945a828096502d9dIra Snyder		chan_dbg(chan, "out of memory for link descriptor\n");
4439c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder		return NULL;
444173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	}
445173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
4469c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	memset(desc, 0, sizeof(*desc));
4479c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	INIT_LIST_HEAD(&desc->tx_list);
4489c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);
4499c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	desc->async_tx.tx_submit = fsl_dma_tx_submit;
4509c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	desc->async_tx.phys = pdesc;
4519c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder
4520ab09c36818ca88f65c88f4d8c6d067fbf10578dIra Snyder#ifdef FSL_DMA_LD_DEBUG
4530ab09c36818ca88f65c88f4d8c6d067fbf10578dIra Snyder	chan_dbg(chan, "LD %p allocated\n", desc);
4540ab09c36818ca88f65c88f4d8c6d067fbf10578dIra Snyder#endif
4550ab09c36818ca88f65c88f4d8c6d067fbf10578dIra Snyder
4569c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	return desc;
457173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
458173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
459173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/**
460173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * fsl_dma_alloc_chan_resources - Allocate resources for DMA channel.
461a1c03319018061304be28d131073ac13a5cb86fbIra Snyder * @chan : Freescale DMA channel
462173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei *
463173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * This function will create a dma pool for descriptor allocation.
464173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei *
465173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * Return - The number of descriptors allocated.
466173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */
467a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic int fsl_dma_alloc_chan_resources(struct dma_chan *dchan)
468173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
469a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	struct fsldma_chan *chan = to_fsl_chan(dchan);
47077cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi
47177cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi	/* Has this channel already been allocated? */
472a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	if (chan->desc_pool)
47377cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi		return 1;
474173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
4759c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	/*
4769c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 * We need the descriptor to be aligned to 32bytes
477173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	 * for meeting FSL DMA specification requirement.
478173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	 */
479b158471ef63bf399165db96e945a828096502d9dIra Snyder	chan->desc_pool = dma_pool_create(chan->name, chan->dev,
4809c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder					  sizeof(struct fsl_desc_sw),
4819c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder					  __alignof__(struct fsl_desc_sw), 0);
482a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	if (!chan->desc_pool) {
483b158471ef63bf399165db96e945a828096502d9dIra Snyder		chan_err(chan, "unable to allocate descriptor pool\n");
4849c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder		return -ENOMEM;
485173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	}
486173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
4879c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	/* there is at least one descriptor free to be allocated */
488173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	return 1;
489173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
490173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
491173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/**
4929c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * fsldma_free_desc_list - Free all descriptors in a queue
4939c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * @chan: Freescae DMA channel
4949c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * @list: the list to free
4959c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder *
4969c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * LOCKING: must hold chan->desc_lock
4979c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder */
4989c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyderstatic void fsldma_free_desc_list(struct fsldma_chan *chan,
4999c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder				  struct list_head *list)
5009c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder{
5019c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	struct fsl_desc_sw *desc, *_desc;
5029c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder
5039c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	list_for_each_entry_safe(desc, _desc, list, node) {
5049c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder		list_del(&desc->node);
5050ab09c36818ca88f65c88f4d8c6d067fbf10578dIra Snyder#ifdef FSL_DMA_LD_DEBUG
5060ab09c36818ca88f65c88f4d8c6d067fbf10578dIra Snyder		chan_dbg(chan, "LD %p free\n", desc);
5070ab09c36818ca88f65c88f4d8c6d067fbf10578dIra Snyder#endif
5089c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder		dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);
5099c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	}
5109c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder}
5119c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder
5129c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyderstatic void fsldma_free_desc_list_reverse(struct fsldma_chan *chan,
5139c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder					  struct list_head *list)
5149c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder{
5159c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	struct fsl_desc_sw *desc, *_desc;
5169c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder
5179c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	list_for_each_entry_safe_reverse(desc, _desc, list, node) {
5189c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder		list_del(&desc->node);
5190ab09c36818ca88f65c88f4d8c6d067fbf10578dIra Snyder#ifdef FSL_DMA_LD_DEBUG
5200ab09c36818ca88f65c88f4d8c6d067fbf10578dIra Snyder		chan_dbg(chan, "LD %p free\n", desc);
5210ab09c36818ca88f65c88f4d8c6d067fbf10578dIra Snyder#endif
5229c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder		dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);
5239c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	}
5249c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder}
5259c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder
5269c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder/**
527173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * fsl_dma_free_chan_resources - Free all resources of the channel.
528a1c03319018061304be28d131073ac13a5cb86fbIra Snyder * @chan : Freescale DMA channel
529173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */
530a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void fsl_dma_free_chan_resources(struct dma_chan *dchan)
531173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
532a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	struct fsldma_chan *chan = to_fsl_chan(dchan);
533173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	unsigned long flags;
534173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
535b158471ef63bf399165db96e945a828096502d9dIra Snyder	chan_dbg(chan, "free all channel resources\n");
536a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	spin_lock_irqsave(&chan->desc_lock, flags);
5379c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	fsldma_free_desc_list(chan, &chan->ld_pending);
5389c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	fsldma_free_desc_list(chan, &chan->ld_running);
539a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	spin_unlock_irqrestore(&chan->desc_lock, flags);
54077cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi
5419c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	dma_pool_destroy(chan->desc_pool);
542a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	chan->desc_pool = NULL;
543173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
544173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
5452187c269ad29510f1d65ec684133d1d3426d0eedZhang Weistatic struct dma_async_tx_descriptor *
546a1c03319018061304be28d131073ac13a5cb86fbIra Snyderfsl_dma_prep_interrupt(struct dma_chan *dchan, unsigned long flags)
5472187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei{
548a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	struct fsldma_chan *chan;
5492187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei	struct fsl_desc_sw *new;
5502187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei
551a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	if (!dchan)
5522187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei		return NULL;
5532187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei
554a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	chan = to_fsl_chan(dchan);
5552187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei
556a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	new = fsl_dma_alloc_descriptor(chan);
5572187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei	if (!new) {
558b158471ef63bf399165db96e945a828096502d9dIra Snyder		chan_err(chan, "%s\n", msg_ld_oom);
5592187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei		return NULL;
5602187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei	}
5612187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei
5622187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei	new->async_tx.cookie = -EBUSY;
563636bdeaa1243327501edfd2a597ed7443eb4239aDan Williams	new->async_tx.flags = flags;
5642187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei
565f79abb627f033c85a6088231f20c85bc4a9bd757Zhang Wei	/* Insert the link descriptor to the LD ring */
566eda34234578fd822c950fd06b5c5ff7ac08b3001Dan Williams	list_add_tail(&new->node, &new->tx_list);
567f79abb627f033c85a6088231f20c85bc4a9bd757Zhang Wei
56831f4306c83a2daa3e348056b720de511bffe5a9bIra Snyder	/* Set End-of-link to the last link descriptor of new list */
569a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	set_ld_eol(chan, new);
5702187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei
5712187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei	return &new->async_tx;
5722187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei}
5732187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei
57431f4306c83a2daa3e348056b720de511bffe5a9bIra Snyderstatic struct dma_async_tx_descriptor *
57531f4306c83a2daa3e348056b720de511bffe5a9bIra Snyderfsl_dma_prep_memcpy(struct dma_chan *dchan,
57631f4306c83a2daa3e348056b720de511bffe5a9bIra Snyder	dma_addr_t dma_dst, dma_addr_t dma_src,
577173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	size_t len, unsigned long flags)
578173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
579a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	struct fsldma_chan *chan;
580173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	struct fsl_desc_sw *first = NULL, *prev = NULL, *new;
581173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	size_t copy;
582173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
583a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	if (!dchan)
584173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		return NULL;
585173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
586173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	if (!len)
587173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		return NULL;
588173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
589a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	chan = to_fsl_chan(dchan);
590173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
591173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	do {
592173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
593173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		/* Allocate the link descriptor from DMA pool */
594a1c03319018061304be28d131073ac13a5cb86fbIra Snyder		new = fsl_dma_alloc_descriptor(chan);
595173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		if (!new) {
596b158471ef63bf399165db96e945a828096502d9dIra Snyder			chan_err(chan, "%s\n", msg_ld_oom);
5972e077f8e8337e52eef3c39c24c31e103b11a0326Ira Snyder			goto fail;
598173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		}
599173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
60056822843ff99c88c778a614851328fcbb1503d10Zhang Wei		copy = min(len, (size_t)FSL_DMA_BCR_MAX_CNT);
601173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
602a1c03319018061304be28d131073ac13a5cb86fbIra Snyder		set_desc_cnt(chan, &new->hw, copy);
603a1c03319018061304be28d131073ac13a5cb86fbIra Snyder		set_desc_src(chan, &new->hw, dma_src);
604a1c03319018061304be28d131073ac13a5cb86fbIra Snyder		set_desc_dst(chan, &new->hw, dma_dst);
605173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
606173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		if (!first)
607173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei			first = new;
608173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		else
609a1c03319018061304be28d131073ac13a5cb86fbIra Snyder			set_desc_next(chan, &prev->hw, new->async_tx.phys);
610173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
611173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		new->async_tx.cookie = 0;
612636bdeaa1243327501edfd2a597ed7443eb4239aDan Williams		async_tx_ack(&new->async_tx);
613173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
614173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		prev = new;
615173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		len -= copy;
616173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		dma_src += copy;
617738f5f7e1ae876448cb7d9c82bea258b69386647Ira Snyder		dma_dst += copy;
618173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
619173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		/* Insert the link descriptor to the LD ring */
620eda34234578fd822c950fd06b5c5ff7ac08b3001Dan Williams		list_add_tail(&new->node, &first->tx_list);
621173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	} while (len);
622173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
623636bdeaa1243327501edfd2a597ed7443eb4239aDan Williams	new->async_tx.flags = flags; /* client is in control of this ack */
624173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	new->async_tx.cookie = -EBUSY;
625173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
62631f4306c83a2daa3e348056b720de511bffe5a9bIra Snyder	/* Set End-of-link to the last link descriptor of new list */
627a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	set_ld_eol(chan, new);
628173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
6292e077f8e8337e52eef3c39c24c31e103b11a0326Ira Snyder	return &first->async_tx;
6302e077f8e8337e52eef3c39c24c31e103b11a0326Ira Snyder
6312e077f8e8337e52eef3c39c24c31e103b11a0326Ira Snyderfail:
6322e077f8e8337e52eef3c39c24c31e103b11a0326Ira Snyder	if (!first)
6332e077f8e8337e52eef3c39c24c31e103b11a0326Ira Snyder		return NULL;
6342e077f8e8337e52eef3c39c24c31e103b11a0326Ira Snyder
6359c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	fsldma_free_desc_list_reverse(chan, &first->tx_list);
6362e077f8e8337e52eef3c39c24c31e103b11a0326Ira Snyder	return NULL;
637173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
638173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
639c14330417ef2050f4bf38ac20e125785fea14351Ira Snyderstatic struct dma_async_tx_descriptor *fsl_dma_prep_sg(struct dma_chan *dchan,
640c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	struct scatterlist *dst_sg, unsigned int dst_nents,
641c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	struct scatterlist *src_sg, unsigned int src_nents,
642c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	unsigned long flags)
643c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder{
644c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	struct fsl_desc_sw *first = NULL, *prev = NULL, *new = NULL;
645c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	struct fsldma_chan *chan = to_fsl_chan(dchan);
646c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	size_t dst_avail, src_avail;
647c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	dma_addr_t dst, src;
648c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	size_t len;
649c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
650c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	/* basic sanity checks */
651c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	if (dst_nents == 0 || src_nents == 0)
652c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		return NULL;
653c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
654c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	if (dst_sg == NULL || src_sg == NULL)
655c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		return NULL;
656c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
657c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	/*
658c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	 * TODO: should we check that both scatterlists have the same
659c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	 * TODO: number of bytes in total? Is that really an error?
660c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	 */
661c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
662c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	/* get prepared for the loop */
663c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	dst_avail = sg_dma_len(dst_sg);
664c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	src_avail = sg_dma_len(src_sg);
665c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
666c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	/* run until we are out of scatterlist entries */
667c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	while (true) {
668c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
669c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		/* create the largest transaction possible */
670c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		len = min_t(size_t, src_avail, dst_avail);
671c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		len = min_t(size_t, len, FSL_DMA_BCR_MAX_CNT);
672c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		if (len == 0)
673c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder			goto fetch;
674c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
675c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		dst = sg_dma_address(dst_sg) + sg_dma_len(dst_sg) - dst_avail;
676c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		src = sg_dma_address(src_sg) + sg_dma_len(src_sg) - src_avail;
677c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
678c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		/* allocate and populate the descriptor */
679c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		new = fsl_dma_alloc_descriptor(chan);
680c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		if (!new) {
681b158471ef63bf399165db96e945a828096502d9dIra Snyder			chan_err(chan, "%s\n", msg_ld_oom);
682c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder			goto fail;
683c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		}
684c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
685c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		set_desc_cnt(chan, &new->hw, len);
686c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		set_desc_src(chan, &new->hw, src);
687c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		set_desc_dst(chan, &new->hw, dst);
688c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
689c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		if (!first)
690c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder			first = new;
691c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		else
692c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder			set_desc_next(chan, &prev->hw, new->async_tx.phys);
693c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
694c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		new->async_tx.cookie = 0;
695c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		async_tx_ack(&new->async_tx);
696c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		prev = new;
697c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
698c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		/* Insert the link descriptor to the LD ring */
699c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		list_add_tail(&new->node, &first->tx_list);
700c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
701c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		/* update metadata */
702c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		dst_avail -= len;
703c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		src_avail -= len;
704c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
705c14330417ef2050f4bf38ac20e125785fea14351Ira Snyderfetch:
706c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		/* fetch the next dst scatterlist entry */
707c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		if (dst_avail == 0) {
708c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
709c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder			/* no more entries: we're done */
710c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder			if (dst_nents == 0)
711c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder				break;
712c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
713c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder			/* fetch the next entry: if there are no more: done */
714c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder			dst_sg = sg_next(dst_sg);
715c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder			if (dst_sg == NULL)
716c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder				break;
717c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
718c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder			dst_nents--;
719c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder			dst_avail = sg_dma_len(dst_sg);
720c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		}
721c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
722c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		/* fetch the next src scatterlist entry */
723c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		if (src_avail == 0) {
724c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
725c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder			/* no more entries: we're done */
726c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder			if (src_nents == 0)
727c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder				break;
728c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
729c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder			/* fetch the next entry: if there are no more: done */
730c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder			src_sg = sg_next(src_sg);
731c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder			if (src_sg == NULL)
732c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder				break;
733c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
734c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder			src_nents--;
735c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder			src_avail = sg_dma_len(src_sg);
736c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		}
737c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	}
738c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
739c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	new->async_tx.flags = flags; /* client is in control of this ack */
740c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	new->async_tx.cookie = -EBUSY;
741c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
742c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	/* Set End-of-link to the last link descriptor of new list */
743c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	set_ld_eol(chan, new);
744c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
745c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	return &first->async_tx;
746c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
747c14330417ef2050f4bf38ac20e125785fea14351Ira Snyderfail:
748c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	if (!first)
749c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder		return NULL;
750c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
751c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	fsldma_free_desc_list_reverse(chan, &first->tx_list);
752c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	return NULL;
753c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder}
754c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder
755173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/**
756bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder * fsl_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
757bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder * @chan: DMA channel
758bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder * @sgl: scatterlist to transfer to/from
759bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder * @sg_len: number of entries in @scatterlist
760bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder * @direction: DMA direction
761bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder * @flags: DMAEngine flags
762185ecb5f4fd43911c35956d4cc7d94a1da30417fAlexandre Bounine * @context: transaction context (ignored)
763bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder *
764bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder * Prepare a set of descriptors for a DMA_SLAVE transaction. Following the
765bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder * DMA_SLAVE API, this gets the device-specific information from the
766bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder * chan->private variable.
767bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder */
768bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyderstatic struct dma_async_tx_descriptor *fsl_dma_prep_slave_sg(
769a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	struct dma_chan *dchan, struct scatterlist *sgl, unsigned int sg_len,
770185ecb5f4fd43911c35956d4cc7d94a1da30417fAlexandre Bounine	enum dma_transfer_direction direction, unsigned long flags,
771185ecb5f4fd43911c35956d4cc7d94a1da30417fAlexandre Bounine	void *context)
772bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder{
773bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder	/*
774968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder	 * This operation is not supported on the Freescale DMA controller
775bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder	 *
776968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder	 * However, we need to provide the function pointer to allow the
777968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder	 * device_control() method to work.
778bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder	 */
779bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder	return NULL;
780bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder}
781bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder
782c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleijstatic int fsl_dma_device_control(struct dma_chan *dchan,
783058276303dbc4ed089c1f7dad0871810b1f5ddf1Linus Walleij				  enum dma_ctrl_cmd cmd, unsigned long arg)
784bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder{
785968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder	struct dma_slave_config *config;
786a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	struct fsldma_chan *chan;
787bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder	unsigned long flags;
788968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder	int size;
789c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij
790a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	if (!dchan)
791c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij		return -EINVAL;
792bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder
793a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	chan = to_fsl_chan(dchan);
794bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder
795968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder	switch (cmd) {
796968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder	case DMA_TERMINATE_ALL:
797f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder		spin_lock_irqsave(&chan->desc_lock, flags);
798f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder
799968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder		/* Halt the DMA engine */
800968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder		dma_halt(chan);
801bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder
802968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder		/* Remove and free all of the descriptors in the LD queue */
803968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder		fsldma_free_desc_list(chan, &chan->ld_pending);
804968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder		fsldma_free_desc_list(chan, &chan->ld_running);
805f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder		chan->idle = true;
806bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder
807968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder		spin_unlock_irqrestore(&chan->desc_lock, flags);
808968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder		return 0;
809968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder
810968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder	case DMA_SLAVE_CONFIG:
811968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder		config = (struct dma_slave_config *)arg;
812968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder
813968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder		/* make sure the channel supports setting burst size */
814968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder		if (!chan->set_request_count)
815968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder			return -ENXIO;
816968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder
817968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder		/* we set the controller burst size depending on direction */
818db8196df4bb6f117caa163aa73b0f16fd62290bdVinod Koul		if (config->direction == DMA_MEM_TO_DEV)
819968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder			size = config->dst_addr_width * config->dst_maxburst;
820968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder		else
821968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder			size = config->src_addr_width * config->src_maxburst;
822968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder
823968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder		chan->set_request_count(chan, size);
824968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder		return 0;
825968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder
826968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder	case FSLDMA_EXTERNAL_START:
827968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder
828968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder		/* make sure the channel supports external start */
829968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder		if (!chan->toggle_ext_start)
830968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder			return -ENXIO;
831968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder
832968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder		chan->toggle_ext_start(chan, arg);
833968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder		return 0;
834968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder
835968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder	default:
836968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder		return -ENXIO;
837968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder	}
838c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij
839c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij	return 0;
840bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder}
841bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder
842bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder/**
8439c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder * fsldma_cleanup_descriptor - cleanup and free a single link descriptor
8449c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * @chan: Freescale DMA channel
8459c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder * @desc: descriptor to cleanup and free
846173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei *
8479c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder * This function is used on a descriptor which has been executed by the DMA
8489c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder * controller. It will run any callbacks, submit any dependencies, and then
8499c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder * free the descriptor.
850173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */
8519c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyderstatic void fsldma_cleanup_descriptor(struct fsldma_chan *chan,
8529c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder				      struct fsl_desc_sw *desc)
853173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
8549c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	struct dma_async_tx_descriptor *txd = &desc->async_tx;
8559c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	struct device *dev = chan->common.device->dev;
8569c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	dma_addr_t src = get_desc_src(chan, desc);
8579c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	dma_addr_t dst = get_desc_dst(chan, desc);
8589c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	u32 len = get_desc_cnt(chan, desc);
8599c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder
8609c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	/* Run the link descriptor callback function */
8619c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	if (txd->callback) {
8629c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder#ifdef FSL_DMA_LD_DEBUG
8639c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder		chan_dbg(chan, "LD %p callback\n", desc);
8649c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder#endif
8659c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder		txd->callback(txd->callback_param);
8669c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	}
867173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
8689c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	/* Run any dependencies */
8699c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	dma_run_dependencies(txd);
870173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
8719c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	/* Unmap the dst buffer, if requested */
8729c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
8739c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder		if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
8749c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder			dma_unmap_single(dev, dst, len, DMA_FROM_DEVICE);
8759c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder		else
8769c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder			dma_unmap_page(dev, dst, len, DMA_FROM_DEVICE);
8779c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	}
8789c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder
8799c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	/* Unmap the src buffer, if requested */
8809c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
8819c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder		if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
8829c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder			dma_unmap_single(dev, src, len, DMA_TO_DEVICE);
8839c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder		else
8849c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder			dma_unmap_page(dev, src, len, DMA_TO_DEVICE);
885173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	}
8869c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder
8879c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder#ifdef FSL_DMA_LD_DEBUG
8889c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	chan_dbg(chan, "LD %p free\n", desc);
8899c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder#endif
8909c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder	dma_pool_free(chan->desc_pool, desc, txd->phys);
891173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
892173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
893173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/**
8949c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * fsl_chan_xfer_ld_queue - transfer any pending transactions
895a1c03319018061304be28d131073ac13a5cb86fbIra Snyder * @chan : Freescale DMA channel
8969c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder *
897f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder * HARDWARE STATE: idle
898dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder * LOCKING: must hold chan->desc_lock
899173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */
900a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void fsl_chan_xfer_ld_queue(struct fsldma_chan *chan)
901173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
9029c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	struct fsl_desc_sw *desc;
903138ef0185177a6d221d24b6aa8f12d867fbbef90Ira Snyder
9049c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	/*
9059c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 * If the list of pending descriptors is empty, then we
9069c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 * don't need to do any work at all
9079c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 */
9089c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	if (list_empty(&chan->ld_pending)) {
909b158471ef63bf399165db96e945a828096502d9dIra Snyder		chan_dbg(chan, "no pending LDs\n");
910dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder		return;
9119c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	}
912173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
9139c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	/*
914f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	 * The DMA controller is not idle, which means that the interrupt
915f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	 * handler will start any queued transactions when it runs after
916f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	 * this transaction finishes
9179c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 */
918f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	if (!chan->idle) {
919b158471ef63bf399165db96e945a828096502d9dIra Snyder		chan_dbg(chan, "DMA controller still busy\n");
920dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder		return;
9219c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	}
9229c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder
9239c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	/*
9249c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 * If there are some link descriptors which have not been
9259c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 * transferred, we need to start the controller
926173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	 */
927173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
9289c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	/*
9299c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 * Move all elements from the queue of pending transactions
9309c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 * onto the list of running transactions
9319c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 */
932f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	chan_dbg(chan, "idle, starting controller\n");
9339c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	desc = list_first_entry(&chan->ld_pending, struct fsl_desc_sw, node);
9349c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	list_splice_tail_init(&chan->ld_pending, &chan->ld_running);
9359c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder
9369c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	/*
937f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	 * The 85xx DMA controller doesn't clear the channel start bit
938f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	 * automatically at the end of a transfer. Therefore we must clear
939f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	 * it in software before starting the transfer.
940f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	 */
941f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) {
942f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder		u32 mode;
943f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder
944f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder		mode = DMA_IN(chan, &chan->regs->mr, 32);
945f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder		mode &= ~FSL_DMA_MR_CS;
946f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder		DMA_OUT(chan, &chan->regs->mr, mode, 32);
947f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	}
948f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder
949f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	/*
9509c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 * Program the descriptor's address into the DMA controller,
9519c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 * then start the DMA transaction
9529c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 */
9539c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	set_cdar(chan, desc->async_tx.phys);
954f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	get_cdar(chan);
955138ef0185177a6d221d24b6aa8f12d867fbbef90Ira Snyder
9569c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	dma_start(chan);
957f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	chan->idle = false;
958173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
959173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
960173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/**
961173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * fsl_dma_memcpy_issue_pending - Issue the DMA start command
962a1c03319018061304be28d131073ac13a5cb86fbIra Snyder * @chan : Freescale DMA channel
963173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */
964a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void fsl_dma_memcpy_issue_pending(struct dma_chan *dchan)
965173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
966a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	struct fsldma_chan *chan = to_fsl_chan(dchan);
967dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	unsigned long flags;
968dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder
969dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	spin_lock_irqsave(&chan->desc_lock, flags);
970a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	fsl_chan_xfer_ld_queue(chan);
971dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	spin_unlock_irqrestore(&chan->desc_lock, flags);
972173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
973173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
974173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/**
9750793448187643b50af89d36b08470baf45a3cab4Linus Walleij * fsl_tx_status - Determine the DMA status
976a1c03319018061304be28d131073ac13a5cb86fbIra Snyder * @chan : Freescale DMA channel
977173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */
9780793448187643b50af89d36b08470baf45a3cab4Linus Walleijstatic enum dma_status fsl_tx_status(struct dma_chan *dchan,
979173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei					dma_cookie_t cookie,
9800793448187643b50af89d36b08470baf45a3cab4Linus Walleij					struct dma_tx_state *txstate)
981173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
982a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	struct fsldma_chan *chan = to_fsl_chan(dchan);
98396a2af41c78b1fbb1f567a3486bdc63f7b31c5fdRussell King - ARM Linux	enum dma_status ret;
984f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	unsigned long flags;
985173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
986f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	spin_lock_irqsave(&chan->desc_lock, flags);
98796a2af41c78b1fbb1f567a3486bdc63f7b31c5fdRussell King - ARM Linux	ret = dma_cookie_status(dchan, cookie, txstate);
988f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	spin_unlock_irqrestore(&chan->desc_lock, flags);
989173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
99096a2af41c78b1fbb1f567a3486bdc63f7b31c5fdRussell King - ARM Linux	return ret;
991173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
992173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
993d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder/*----------------------------------------------------------------------------*/
994d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder/* Interrupt Handling                                                         */
995d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder/*----------------------------------------------------------------------------*/
996d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder
997e7a29151de1bd52081f27f149b68074fac0323beIra Snyderstatic irqreturn_t fsldma_chan_irq(int irq, void *data)
998173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
999a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	struct fsldma_chan *chan = data;
1000a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	u32 stat;
1001173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
10029c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	/* save and clear the status register */
1003a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	stat = get_sr(chan);
10049c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	set_sr(chan, stat);
1005b158471ef63bf399165db96e945a828096502d9dIra Snyder	chan_dbg(chan, "irq: stat = 0x%x\n", stat);
1006173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1007f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	/* check that this was really our device */
1008173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	stat &= ~(FSL_DMA_SR_CB | FSL_DMA_SR_CH);
1009173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	if (!stat)
1010173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		return IRQ_NONE;
1011173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1012173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	if (stat & FSL_DMA_SR_TE)
1013b158471ef63bf399165db96e945a828096502d9dIra Snyder		chan_err(chan, "Transfer Error!\n");
1014173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
10159c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	/*
10169c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 * Programming Error
1017f79abb627f033c85a6088231f20c85bc4a9bd757Zhang Wei	 * The DMA_INTERRUPT async_tx is a NULL transfer, which will
1018f79abb627f033c85a6088231f20c85bc4a9bd757Zhang Wei	 * triger a PE interrupt.
1019f79abb627f033c85a6088231f20c85bc4a9bd757Zhang Wei	 */
1020f79abb627f033c85a6088231f20c85bc4a9bd757Zhang Wei	if (stat & FSL_DMA_SR_PE) {
1021b158471ef63bf399165db96e945a828096502d9dIra Snyder		chan_dbg(chan, "irq: Programming Error INT\n");
1022f79abb627f033c85a6088231f20c85bc4a9bd757Zhang Wei		stat &= ~FSL_DMA_SR_PE;
1023f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder		if (get_bcr(chan) != 0)
1024f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder			chan_err(chan, "Programming Error!\n");
10251c62979ed29a8e2bf9fbe1db101c81a0089676f8Zhang Wei	}
10261c62979ed29a8e2bf9fbe1db101c81a0089676f8Zhang Wei
10279c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	/*
10289c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 * For MPC8349, EOCDI event need to update cookie
10291c62979ed29a8e2bf9fbe1db101c81a0089676f8Zhang Wei	 * and start the next transfer if it exist.
10301c62979ed29a8e2bf9fbe1db101c81a0089676f8Zhang Wei	 */
10311c62979ed29a8e2bf9fbe1db101c81a0089676f8Zhang Wei	if (stat & FSL_DMA_SR_EOCDI) {
1032b158471ef63bf399165db96e945a828096502d9dIra Snyder		chan_dbg(chan, "irq: End-of-Chain link INT\n");
10331c62979ed29a8e2bf9fbe1db101c81a0089676f8Zhang Wei		stat &= ~FSL_DMA_SR_EOCDI;
1034173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	}
1035173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
10369c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	/*
10379c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	 * If it current transfer is the end-of-transfer,
1038173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	 * we should clear the Channel Start bit for
1039173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	 * prepare next transfer.
1040173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	 */
10411c62979ed29a8e2bf9fbe1db101c81a0089676f8Zhang Wei	if (stat & FSL_DMA_SR_EOLNI) {
1042b158471ef63bf399165db96e945a828096502d9dIra Snyder		chan_dbg(chan, "irq: End-of-link INT\n");
1043173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		stat &= ~FSL_DMA_SR_EOLNI;
1044173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	}
1045173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1046f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	/* check that the DMA controller is really idle */
1047f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	if (!dma_is_idle(chan))
1048f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder		chan_err(chan, "irq: controller not idle!\n");
1049f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder
1050f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	/* check that we handled all of the bits */
1051173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	if (stat)
1052f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder		chan_err(chan, "irq: unhandled sr 0x%08x\n", stat);
1053173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1054f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	/*
1055f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	 * Schedule the tasklet to handle all cleanup of the current
1056f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	 * transaction. It will start a new transaction if there is
1057f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	 * one pending.
1058f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	 */
1059a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	tasklet_schedule(&chan->tasklet);
1060f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	chan_dbg(chan, "irq: Exit\n");
1061173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	return IRQ_HANDLED;
1062173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
1063173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1064d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyderstatic void dma_do_tasklet(unsigned long data)
1065d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder{
1066a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	struct fsldma_chan *chan = (struct fsldma_chan *)data;
1067dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	struct fsl_desc_sw *desc, *_desc;
1068dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	LIST_HEAD(ld_cleanup);
1069f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	unsigned long flags;
1070f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder
1071f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	chan_dbg(chan, "tasklet entry\n");
1072f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder
1073f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	spin_lock_irqsave(&chan->desc_lock, flags);
1074dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder
1075dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	/* update the cookie if we have some descriptors to cleanup */
1076dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	if (!list_empty(&chan->ld_running)) {
1077dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder		dma_cookie_t cookie;
1078dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder
1079dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder		desc = to_fsl_desc(chan->ld_running.prev);
1080dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder		cookie = desc->async_tx.cookie;
1081f7fbce07c6ce26a25b4e0cb5f241c361fde87901Russell King - ARM Linux		dma_cookie_complete(&desc->async_tx);
1082dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder
1083dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder		chan_dbg(chan, "completed_cookie=%d\n", cookie);
1084dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	}
1085dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder
1086dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	/*
1087dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	 * move the descriptors to a temporary list so we can drop the lock
1088dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	 * during the entire cleanup operation
1089dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	 */
1090dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	list_splice_tail_init(&chan->ld_running, &ld_cleanup);
1091dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder
1092dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	/* the hardware is now idle and ready for more */
1093f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	chan->idle = true;
1094f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder
1095dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	/*
1096dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	 * Start any pending transactions automatically
1097dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	 *
1098dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	 * In the ideal case, we keep the DMA controller busy while we go
1099dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	 * ahead and free the descriptors below.
1100dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	 */
1101f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	fsl_chan_xfer_ld_queue(chan);
1102dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	spin_unlock_irqrestore(&chan->desc_lock, flags);
1103dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder
1104dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	/* Run the callback for each descriptor, in order */
1105dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	list_for_each_entry_safe(desc, _desc, &ld_cleanup, node) {
1106dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder
1107dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder		/* Remove from the list of transactions */
1108dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder		list_del(&desc->node);
1109dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder
1110dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder		/* Run all cleanup for this descriptor */
1111dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder		fsldma_cleanup_descriptor(chan, desc);
1112dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder	}
1113dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder
1114f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	chan_dbg(chan, "tasklet exit\n");
1115d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder}
1116d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder
1117d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyderstatic irqreturn_t fsldma_ctrl_irq(int irq, void *data)
1118173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
1119a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder	struct fsldma_device *fdev = data;
1120d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	struct fsldma_chan *chan;
1121d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	unsigned int handled = 0;
1122d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	u32 gsr, mask;
1123d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	int i;
1124173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1125e7a29151de1bd52081f27f149b68074fac0323beIra Snyder	gsr = (fdev->feature & FSL_DMA_BIG_ENDIAN) ? in_be32(fdev->regs)
1126d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder						   : in_le32(fdev->regs);
1127d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	mask = 0xff000000;
1128d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	dev_dbg(fdev->dev, "IRQ: gsr 0x%.8x\n", gsr);
1129173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1130d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
1131d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		chan = fdev->chan[i];
1132d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		if (!chan)
1133d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder			continue;
1134d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder
1135d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		if (gsr & mask) {
1136d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder			dev_dbg(fdev->dev, "IRQ: chan %d\n", chan->id);
1137d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder			fsldma_chan_irq(irq, chan);
1138d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder			handled++;
1139d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		}
1140d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder
1141d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		gsr &= ~mask;
1142d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		mask >>= 8;
1143d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	}
1144d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder
1145d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	return IRQ_RETVAL(handled);
1146173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
1147173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1148d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyderstatic void fsldma_free_irqs(struct fsldma_device *fdev)
1149173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
1150d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	struct fsldma_chan *chan;
1151d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	int i;
1152d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder
1153d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	if (fdev->irq != NO_IRQ) {
1154d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		dev_dbg(fdev->dev, "free per-controller IRQ\n");
1155d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		free_irq(fdev->irq, fdev);
1156d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		return;
1157d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	}
1158d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder
1159d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
1160d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		chan = fdev->chan[i];
1161d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		if (chan && chan->irq != NO_IRQ) {
1162b158471ef63bf399165db96e945a828096502d9dIra Snyder			chan_dbg(chan, "free per-channel IRQ\n");
1163d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder			free_irq(chan->irq, chan);
1164d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		}
1165d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	}
1166d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder}
1167d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder
1168d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyderstatic int fsldma_request_irqs(struct fsldma_device *fdev)
1169d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder{
1170d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	struct fsldma_chan *chan;
1171d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	int ret;
1172d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	int i;
1173d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder
1174d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	/* if we have a per-controller IRQ, use that */
1175d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	if (fdev->irq != NO_IRQ) {
1176d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		dev_dbg(fdev->dev, "request per-controller IRQ\n");
1177d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		ret = request_irq(fdev->irq, fsldma_ctrl_irq, IRQF_SHARED,
1178d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder				  "fsldma-controller", fdev);
1179d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		return ret;
1180d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	}
1181d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder
1182d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	/* no per-controller IRQ, use the per-channel IRQs */
1183d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
1184d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		chan = fdev->chan[i];
1185d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		if (!chan)
1186d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder			continue;
1187d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder
1188d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		if (chan->irq == NO_IRQ) {
1189b158471ef63bf399165db96e945a828096502d9dIra Snyder			chan_err(chan, "interrupts property missing in device tree\n");
1190d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder			ret = -ENODEV;
1191d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder			goto out_unwind;
1192d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		}
1193d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder
1194b158471ef63bf399165db96e945a828096502d9dIra Snyder		chan_dbg(chan, "request per-channel IRQ\n");
1195d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		ret = request_irq(chan->irq, fsldma_chan_irq, IRQF_SHARED,
1196d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder				  "fsldma-chan", chan);
1197d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		if (ret) {
1198b158471ef63bf399165db96e945a828096502d9dIra Snyder			chan_err(chan, "unable to request per-channel IRQ\n");
1199d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder			goto out_unwind;
1200d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		}
1201d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	}
1202d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder
1203d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	return 0;
1204d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder
1205d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyderout_unwind:
1206d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	for (/* none */; i >= 0; i--) {
1207d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		chan = fdev->chan[i];
1208d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		if (!chan)
1209d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder			continue;
1210d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder
1211d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		if (chan->irq == NO_IRQ)
1212d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder			continue;
1213d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder
1214d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		free_irq(chan->irq, chan);
1215d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	}
1216d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder
1217d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	return ret;
1218173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
1219173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1220a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder/*----------------------------------------------------------------------------*/
1221a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder/* OpenFirmware Subsystem                                                     */
1222a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder/*----------------------------------------------------------------------------*/
1223a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder
1224a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyderstatic int __devinit fsl_dma_chan_probe(struct fsldma_device *fdev,
122577cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi	struct device_node *node, u32 feature, const char *compatible)
1226173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
1227a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	struct fsldma_chan *chan;
12284ce0e953f6286777452bf07c83056342d6b9b257Ira Snyder	struct resource res;
1229173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	int err;
1230173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1231173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	/* alloc channel */
1232a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	chan = kzalloc(sizeof(*chan), GFP_KERNEL);
1233a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	if (!chan) {
1234e7a29151de1bd52081f27f149b68074fac0323beIra Snyder		dev_err(fdev->dev, "no free memory for DMA channels!\n");
1235e7a29151de1bd52081f27f149b68074fac0323beIra Snyder		err = -ENOMEM;
1236e7a29151de1bd52081f27f149b68074fac0323beIra Snyder		goto out_return;
1237e7a29151de1bd52081f27f149b68074fac0323beIra Snyder	}
1238e7a29151de1bd52081f27f149b68074fac0323beIra Snyder
1239e7a29151de1bd52081f27f149b68074fac0323beIra Snyder	/* ioremap registers for use */
1240a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	chan->regs = of_iomap(node, 0);
1241a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	if (!chan->regs) {
1242e7a29151de1bd52081f27f149b68074fac0323beIra Snyder		dev_err(fdev->dev, "unable to ioremap registers\n");
1243e7a29151de1bd52081f27f149b68074fac0323beIra Snyder		err = -ENOMEM;
1244a1c03319018061304be28d131073ac13a5cb86fbIra Snyder		goto out_free_chan;
1245173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	}
1246173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
12474ce0e953f6286777452bf07c83056342d6b9b257Ira Snyder	err = of_address_to_resource(node, 0, &res);
1248173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	if (err) {
1249e7a29151de1bd52081f27f149b68074fac0323beIra Snyder		dev_err(fdev->dev, "unable to find 'reg' property\n");
1250e7a29151de1bd52081f27f149b68074fac0323beIra Snyder		goto out_iounmap_regs;
1251173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	}
1252173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1253a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	chan->feature = feature;
1254173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	if (!fdev->feature)
1255a1c03319018061304be28d131073ac13a5cb86fbIra Snyder		fdev->feature = chan->feature;
1256173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1257e7a29151de1bd52081f27f149b68074fac0323beIra Snyder	/*
1258e7a29151de1bd52081f27f149b68074fac0323beIra Snyder	 * If the DMA device's feature is different than the feature
1259e7a29151de1bd52081f27f149b68074fac0323beIra Snyder	 * of its channels, report the bug
1260173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	 */
1261a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	WARN_ON(fdev->feature != chan->feature);
1262e7a29151de1bd52081f27f149b68074fac0323beIra Snyder
1263a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	chan->dev = fdev->dev;
1264a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	chan->id = ((res.start - 0x100) & 0xfff) >> 7;
1265a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	if (chan->id >= FSL_DMA_MAX_CHANS_PER_DEVICE) {
1266e7a29151de1bd52081f27f149b68074fac0323beIra Snyder		dev_err(fdev->dev, "too many channels for device\n");
1267173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei		err = -EINVAL;
1268e7a29151de1bd52081f27f149b68074fac0323beIra Snyder		goto out_iounmap_regs;
1269173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	}
1270173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1271a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	fdev->chan[chan->id] = chan;
1272a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	tasklet_init(&chan->tasklet, dma_do_tasklet, (unsigned long)chan);
1273b158471ef63bf399165db96e945a828096502d9dIra Snyder	snprintf(chan->name, sizeof(chan->name), "chan%d", chan->id);
1274e7a29151de1bd52081f27f149b68074fac0323beIra Snyder
1275e7a29151de1bd52081f27f149b68074fac0323beIra Snyder	/* Initialize the channel */
1276a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	dma_init(chan);
1277173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1278173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	/* Clear cdar registers */
1279a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	set_cdar(chan, 0);
1280173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1281a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	switch (chan->feature & FSL_DMA_IP_MASK) {
1282173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	case FSL_DMA_IP_85XX:
1283a1c03319018061304be28d131073ac13a5cb86fbIra Snyder		chan->toggle_ext_pause = fsl_chan_toggle_ext_pause;
1284173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	case FSL_DMA_IP_83XX:
1285a1c03319018061304be28d131073ac13a5cb86fbIra Snyder		chan->toggle_ext_start = fsl_chan_toggle_ext_start;
1286a1c03319018061304be28d131073ac13a5cb86fbIra Snyder		chan->set_src_loop_size = fsl_chan_set_src_loop_size;
1287a1c03319018061304be28d131073ac13a5cb86fbIra Snyder		chan->set_dst_loop_size = fsl_chan_set_dst_loop_size;
1288a1c03319018061304be28d131073ac13a5cb86fbIra Snyder		chan->set_request_count = fsl_chan_set_request_count;
1289173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	}
1290173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1291a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	spin_lock_init(&chan->desc_lock);
12929c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	INIT_LIST_HEAD(&chan->ld_pending);
12939c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder	INIT_LIST_HEAD(&chan->ld_running);
1294f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder	chan->idle = true;
1295173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1296a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	chan->common.device = &fdev->common;
12978ac695463f37af902e953d575d3f782e32e170daRussell King - ARM Linux	dma_cookie_init(&chan->common);
1298173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1299d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	/* find the IRQ line, if it exists in the device tree */
1300a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	chan->irq = irq_of_parse_and_map(node, 0);
1301d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder
1302173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	/* Add the channel to DMA device channel list */
1303a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	list_add_tail(&chan->common.device_node, &fdev->common.channels);
1304173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	fdev->common.chancnt++;
1305173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1306a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	dev_info(fdev->dev, "#%d (%s), irq %d\n", chan->id, compatible,
1307a1c03319018061304be28d131073ac13a5cb86fbIra Snyder		 chan->irq != NO_IRQ ? chan->irq : fdev->irq);
1308173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1309173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	return 0;
131051ee87f27a1d2c0e08492924f2fb0223c4c704d9Li Yang
1311e7a29151de1bd52081f27f149b68074fac0323beIra Snyderout_iounmap_regs:
1312a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	iounmap(chan->regs);
1313a1c03319018061304be28d131073ac13a5cb86fbIra Snyderout_free_chan:
1314a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	kfree(chan);
1315e7a29151de1bd52081f27f149b68074fac0323beIra Snyderout_return:
1316173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	return err;
1317173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
1318173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1319a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void fsl_dma_chan_remove(struct fsldma_chan *chan)
1320173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
1321a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	irq_dispose_mapping(chan->irq);
1322a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	list_del(&chan->common.device_node);
1323a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	iounmap(chan->regs);
1324a1c03319018061304be28d131073ac13a5cb86fbIra Snyder	kfree(chan);
1325173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
1326173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1327000061245a6797d542854106463b6b20fbdcb12eGrant Likelystatic int __devinit fsldma_of_probe(struct platform_device *op)
1328173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
1329a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder	struct fsldma_device *fdev;
133077cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi	struct device_node *child;
1331e7a29151de1bd52081f27f149b68074fac0323beIra Snyder	int err;
1332173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1333a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder	fdev = kzalloc(sizeof(*fdev), GFP_KERNEL);
1334173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	if (!fdev) {
1335e7a29151de1bd52081f27f149b68074fac0323beIra Snyder		dev_err(&op->dev, "No enough memory for 'priv'\n");
1336e7a29151de1bd52081f27f149b68074fac0323beIra Snyder		err = -ENOMEM;
1337e7a29151de1bd52081f27f149b68074fac0323beIra Snyder		goto out_return;
1338173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	}
1339e7a29151de1bd52081f27f149b68074fac0323beIra Snyder
1340e7a29151de1bd52081f27f149b68074fac0323beIra Snyder	fdev->dev = &op->dev;
1341173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	INIT_LIST_HEAD(&fdev->common.channels);
1342173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1343e7a29151de1bd52081f27f149b68074fac0323beIra Snyder	/* ioremap the registers for use */
134461c7a080a5a061c976988fd4b844dfb468dda255Grant Likely	fdev->regs = of_iomap(op->dev.of_node, 0);
1345e7a29151de1bd52081f27f149b68074fac0323beIra Snyder	if (!fdev->regs) {
1346e7a29151de1bd52081f27f149b68074fac0323beIra Snyder		dev_err(&op->dev, "unable to ioremap registers\n");
1347e7a29151de1bd52081f27f149b68074fac0323beIra Snyder		err = -ENOMEM;
1348e7a29151de1bd52081f27f149b68074fac0323beIra Snyder		goto out_free_fdev;
1349173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	}
1350173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1351d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	/* map the channel IRQ if it exists, but don't hookup the handler yet */
135261c7a080a5a061c976988fd4b844dfb468dda255Grant Likely	fdev->irq = irq_of_parse_and_map(op->dev.of_node, 0);
1353d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder
1354173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask);
1355173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	dma_cap_set(DMA_INTERRUPT, fdev->common.cap_mask);
1356c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	dma_cap_set(DMA_SG, fdev->common.cap_mask);
1357bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder	dma_cap_set(DMA_SLAVE, fdev->common.cap_mask);
1358173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	fdev->common.device_alloc_chan_resources = fsl_dma_alloc_chan_resources;
1359173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	fdev->common.device_free_chan_resources = fsl_dma_free_chan_resources;
13602187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei	fdev->common.device_prep_dma_interrupt = fsl_dma_prep_interrupt;
1361173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	fdev->common.device_prep_dma_memcpy = fsl_dma_prep_memcpy;
1362c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder	fdev->common.device_prep_dma_sg = fsl_dma_prep_sg;
13630793448187643b50af89d36b08470baf45a3cab4Linus Walleij	fdev->common.device_tx_status = fsl_tx_status;
1364173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	fdev->common.device_issue_pending = fsl_dma_memcpy_issue_pending;
1365bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder	fdev->common.device_prep_slave_sg = fsl_dma_prep_slave_sg;
1366c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij	fdev->common.device_control = fsl_dma_device_control;
1367e7a29151de1bd52081f27f149b68074fac0323beIra Snyder	fdev->common.dev = &op->dev;
1368173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1369e2c8e425baa01a4c8e6ae1b90194ed3d3cde0c66Li Yang	dma_set_mask(&(op->dev), DMA_BIT_MASK(36));
1370e2c8e425baa01a4c8e6ae1b90194ed3d3cde0c66Li Yang
1371e7a29151de1bd52081f27f149b68074fac0323beIra Snyder	dev_set_drvdata(&op->dev, fdev);
137277cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi
1373e7a29151de1bd52081f27f149b68074fac0323beIra Snyder	/*
1374e7a29151de1bd52081f27f149b68074fac0323beIra Snyder	 * We cannot use of_platform_bus_probe() because there is no
1375e7a29151de1bd52081f27f149b68074fac0323beIra Snyder	 * of_platform_bus_remove(). Instead, we manually instantiate every DMA
137677cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi	 * channel object.
137777cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi	 */
137861c7a080a5a061c976988fd4b844dfb468dda255Grant Likely	for_each_child_of_node(op->dev.of_node, child) {
1379e7a29151de1bd52081f27f149b68074fac0323beIra Snyder		if (of_device_is_compatible(child, "fsl,eloplus-dma-channel")) {
138077cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi			fsl_dma_chan_probe(fdev, child,
138177cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi				FSL_DMA_IP_85XX | FSL_DMA_BIG_ENDIAN,
138277cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi				"fsl,eloplus-dma-channel");
1383e7a29151de1bd52081f27f149b68074fac0323beIra Snyder		}
1384e7a29151de1bd52081f27f149b68074fac0323beIra Snyder
1385e7a29151de1bd52081f27f149b68074fac0323beIra Snyder		if (of_device_is_compatible(child, "fsl,elo-dma-channel")) {
138677cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi			fsl_dma_chan_probe(fdev, child,
138777cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi				FSL_DMA_IP_83XX | FSL_DMA_LITTLE_ENDIAN,
138877cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi				"fsl,elo-dma-channel");
1389e7a29151de1bd52081f27f149b68074fac0323beIra Snyder		}
139077cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi	}
1391173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1392d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	/*
1393d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	 * Hookup the IRQ handler(s)
1394d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	 *
1395d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	 * If we have a per-controller interrupt, we prefer that to the
1396d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	 * per-channel interrupts to reduce the number of shared interrupt
1397d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	 * handlers on the same IRQ line
1398d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	 */
1399d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	err = fsldma_request_irqs(fdev);
1400d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	if (err) {
1401d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		dev_err(fdev->dev, "unable to request IRQs\n");
1402d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder		goto out_free_fdev;
1403d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	}
1404d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder
1405173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	dma_async_device_register(&fdev->common);
1406173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	return 0;
1407173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1408e7a29151de1bd52081f27f149b68074fac0323beIra Snyderout_free_fdev:
1409d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	irq_dispose_mapping(fdev->irq);
1410173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	kfree(fdev);
1411e7a29151de1bd52081f27f149b68074fac0323beIra Snyderout_return:
1412173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	return err;
1413173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
1414173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
14152dc11581376829303b98eadb2de253bee065a56aGrant Likelystatic int fsldma_of_remove(struct platform_device *op)
141677cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi{
1417a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder	struct fsldma_device *fdev;
141877cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi	unsigned int i;
141977cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi
1420e7a29151de1bd52081f27f149b68074fac0323beIra Snyder	fdev = dev_get_drvdata(&op->dev);
142177cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi	dma_async_device_unregister(&fdev->common);
142277cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi
1423d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder	fsldma_free_irqs(fdev);
1424d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder
1425e7a29151de1bd52081f27f149b68074fac0323beIra Snyder	for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
142677cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi		if (fdev->chan[i])
142777cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi			fsl_dma_chan_remove(fdev->chan[i]);
1428e7a29151de1bd52081f27f149b68074fac0323beIra Snyder	}
142977cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi
1430e7a29151de1bd52081f27f149b68074fac0323beIra Snyder	iounmap(fdev->regs);
1431e7a29151de1bd52081f27f149b68074fac0323beIra Snyder	dev_set_drvdata(&op->dev, NULL);
143277cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi	kfree(fdev);
143377cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi
143477cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi	return 0;
143577cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi}
143677cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi
14374b1cf1facca31b7db2a61d8aa2ba40d5a93a0957Márton Némethstatic const struct of_device_id fsldma_of_ids[] = {
1438049c9d45531d9825bf737891163a794fca1421c5Kumar Gala	{ .compatible = "fsl,eloplus-dma", },
1439049c9d45531d9825bf737891163a794fca1421c5Kumar Gala	{ .compatible = "fsl,elo-dma", },
1440173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei	{}
1441173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei};
1442173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
14438faa7cf828bca1745a4ed599876567f5afc47544Ira W. Snyderstatic struct platform_driver fsldma_of_driver = {
14444018294b53d1dae026880e45f174c1cc63b5d435Grant Likely	.driver = {
14454018294b53d1dae026880e45f174c1cc63b5d435Grant Likely		.name = "fsl-elo-dma",
14464018294b53d1dae026880e45f174c1cc63b5d435Grant Likely		.owner = THIS_MODULE,
14474018294b53d1dae026880e45f174c1cc63b5d435Grant Likely		.of_match_table = fsldma_of_ids,
14484018294b53d1dae026880e45f174c1cc63b5d435Grant Likely	},
14494018294b53d1dae026880e45f174c1cc63b5d435Grant Likely	.probe = fsldma_of_probe,
14504018294b53d1dae026880e45f174c1cc63b5d435Grant Likely	.remove = fsldma_of_remove,
1451173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei};
1452173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1453a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder/*----------------------------------------------------------------------------*/
1454a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder/* Module Init / Exit                                                         */
1455a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder/*----------------------------------------------------------------------------*/
1456a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder
1457a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyderstatic __init int fsldma_init(void)
1458173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{
145977cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi	pr_info("Freescale Elo / Elo Plus DMA driver\n");
1460000061245a6797d542854106463b6b20fbdcb12eGrant Likely	return platform_driver_register(&fsldma_of_driver);
146177cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi}
146277cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi
1463a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyderstatic void __exit fsldma_exit(void)
146477cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi{
1465000061245a6797d542854106463b6b20fbdcb12eGrant Likely	platform_driver_unregister(&fsldma_of_driver);
1466173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}
1467173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei
1468a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snydersubsys_initcall(fsldma_init);
1469a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snydermodule_exit(fsldma_exit);
147077cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi
147177cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur TabiMODULE_DESCRIPTION("Freescale Elo / Elo Plus DMA driver");
147277cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur TabiMODULE_LICENSE("GPL");
1473