1173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/* 2173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * Freescale MPC85xx, MPC83xx DMA Engine support 3173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * 4e2c8e425baa01a4c8e6ae1b90194ed3d3cde0c66Li Yang * Copyright (C) 2007-2010 Freescale Semiconductor, Inc. All rights reserved. 5173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * 6173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * Author: 7173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * Zhang Wei <wei.zhang@freescale.com>, Jul 2007 8173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * Ebony Zhu <ebony.zhu@freescale.com>, May 2007 9173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * 10173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * Description: 11173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * DMA engine driver for Freescale MPC8540 DMA controller, which is 12173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * also fit for MPC8560, MPC8555, MPC8548, MPC8641, and etc. 13c2e07b3a9ced33dd92597201be3931be8ea57ed6Stefan Weil * The support for MPC8349 DMA controller is also added. 14173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * 15a7aea373b4ca428f1be2c1fedd2f26c8e3f2864dIra W. Snyder * This driver instructs the DMA controller to issue the PCI Read Multiple 16a7aea373b4ca428f1be2c1fedd2f26c8e3f2864dIra W. Snyder * command for PCI read operations, instead of using the default PCI Read Line 17a7aea373b4ca428f1be2c1fedd2f26c8e3f2864dIra W. Snyder * command. Please be aware that this setting may result in read pre-fetching 18a7aea373b4ca428f1be2c1fedd2f26c8e3f2864dIra W. Snyder * on some platforms. 19a7aea373b4ca428f1be2c1fedd2f26c8e3f2864dIra W. Snyder * 20173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * This is free software; you can redistribute it and/or modify 21173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * it under the terms of the GNU General Public License as published by 22173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * the Free Software Foundation; either version 2 of the License, or 23173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * (at your option) any later version. 24173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * 25173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */ 26173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 27173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei#include <linux/init.h> 28173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei#include <linux/module.h> 29173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei#include <linux/pci.h> 305a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 31173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei#include <linux/interrupt.h> 32173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei#include <linux/dmaengine.h> 33173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei#include <linux/delay.h> 34173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei#include <linux/dma-mapping.h> 35173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei#include <linux/dmapool.h> 36173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei#include <linux/of_platform.h> 37173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 38d2ebfb335b0426deb1a4fb14e4e926d81ecd8235Russell King - ARM Linux#include "dmaengine.h" 39173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei#include "fsldma.h" 40173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 41b158471ef63bf399165db96e945a828096502d9dIra Snyder#define chan_dbg(chan, fmt, arg...) \ 42b158471ef63bf399165db96e945a828096502d9dIra Snyder dev_dbg(chan->dev, "%s: " fmt, chan->name, ##arg) 43b158471ef63bf399165db96e945a828096502d9dIra Snyder#define chan_err(chan, fmt, arg...) \ 44b158471ef63bf399165db96e945a828096502d9dIra Snyder dev_err(chan->dev, "%s: " fmt, chan->name, ##arg) 45c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 46b158471ef63bf399165db96e945a828096502d9dIra Snyderstatic const char msg_ld_oom[] = "No free memory for link descriptor"; 47173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 48e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder/* 49e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder * Register Helpers 50e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder */ 51173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 52a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void set_sr(struct fsldma_chan *chan, u32 val) 53173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 54a1c03319018061304be28d131073ac13a5cb86fbIra Snyder DMA_OUT(chan, &chan->regs->sr, val, 32); 55173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 56173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 57a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic u32 get_sr(struct fsldma_chan *chan) 58173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 59a1c03319018061304be28d131073ac13a5cb86fbIra Snyder return DMA_IN(chan, &chan->regs->sr, 32); 60173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 61173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 62e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyderstatic void set_cdar(struct fsldma_chan *chan, dma_addr_t addr) 63e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder{ 64e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder DMA_OUT(chan, &chan->regs->cdar, addr | FSL_DMA_SNEN, 64); 65e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder} 66e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder 67e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyderstatic dma_addr_t get_cdar(struct fsldma_chan *chan) 68e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder{ 69e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder return DMA_IN(chan, &chan->regs->cdar, 64) & ~FSL_DMA_SNEN; 70e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder} 71e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder 72e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyderstatic u32 get_bcr(struct fsldma_chan *chan) 73e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder{ 74e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder return DMA_IN(chan, &chan->regs->bcr, 32); 75e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder} 76e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder 77e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder/* 78e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder * Descriptor Helpers 79e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder */ 80e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder 81a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void set_desc_cnt(struct fsldma_chan *chan, 82173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei struct fsl_dma_ld_hw *hw, u32 count) 83173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 84a1c03319018061304be28d131073ac13a5cb86fbIra Snyder hw->count = CPU_TO_DMA(chan, count, 32); 85173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 86173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 879c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyderstatic u32 get_desc_cnt(struct fsldma_chan *chan, struct fsl_desc_sw *desc) 889c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder{ 899c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder return DMA_TO_CPU(chan, desc->hw.count, 32); 909c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder} 919c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder 92a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void set_desc_src(struct fsldma_chan *chan, 9331f4306c83a2daa3e348056b720de511bffe5a9bIra Snyder struct fsl_dma_ld_hw *hw, dma_addr_t src) 94173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 95173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei u64 snoop_bits; 96173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 97a1c03319018061304be28d131073ac13a5cb86fbIra Snyder snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) 98173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei ? ((u64)FSL_DMA_SATR_SREADTYPE_SNOOP_READ << 32) : 0; 99a1c03319018061304be28d131073ac13a5cb86fbIra Snyder hw->src_addr = CPU_TO_DMA(chan, snoop_bits | src, 64); 100173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 101173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1029c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyderstatic dma_addr_t get_desc_src(struct fsldma_chan *chan, 1039c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder struct fsl_desc_sw *desc) 1049c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder{ 1059c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder u64 snoop_bits; 1069c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder 1079c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) 1089c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder ? ((u64)FSL_DMA_SATR_SREADTYPE_SNOOP_READ << 32) : 0; 1099c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder return DMA_TO_CPU(chan, desc->hw.src_addr, 64) & ~snoop_bits; 1109c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder} 1119c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder 112a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void set_desc_dst(struct fsldma_chan *chan, 11331f4306c83a2daa3e348056b720de511bffe5a9bIra Snyder struct fsl_dma_ld_hw *hw, dma_addr_t dst) 114173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 115173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei u64 snoop_bits; 116173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 117a1c03319018061304be28d131073ac13a5cb86fbIra Snyder snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) 118173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei ? ((u64)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE << 32) : 0; 119a1c03319018061304be28d131073ac13a5cb86fbIra Snyder hw->dst_addr = CPU_TO_DMA(chan, snoop_bits | dst, 64); 120173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 121173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1229c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyderstatic dma_addr_t get_desc_dst(struct fsldma_chan *chan, 1239c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder struct fsl_desc_sw *desc) 1249c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder{ 1259c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder u64 snoop_bits; 1269c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder 1279c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) 1289c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder ? ((u64)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE << 32) : 0; 1299c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder return DMA_TO_CPU(chan, desc->hw.dst_addr, 64) & ~snoop_bits; 1309c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder} 1319c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder 132a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void set_desc_next(struct fsldma_chan *chan, 13331f4306c83a2daa3e348056b720de511bffe5a9bIra Snyder struct fsl_dma_ld_hw *hw, dma_addr_t next) 134173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 135173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei u64 snoop_bits; 136173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 137a1c03319018061304be28d131073ac13a5cb86fbIra Snyder snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX) 138173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei ? FSL_DMA_SNEN : 0; 139a1c03319018061304be28d131073ac13a5cb86fbIra Snyder hw->next_ln_addr = CPU_TO_DMA(chan, snoop_bits | next, 64); 140173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 141173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 14231f4306c83a2daa3e348056b720de511bffe5a9bIra Snyderstatic void set_ld_eol(struct fsldma_chan *chan, struct fsl_desc_sw *desc) 143173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 144e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder u64 snoop_bits; 145173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 146e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX) 147e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder ? FSL_DMA_SNEN : 0; 148173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 149e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder desc->hw.next_ln_addr = CPU_TO_DMA(chan, 150e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder DMA_TO_CPU(chan, desc->hw.next_ln_addr, 64) | FSL_DMA_EOL 151e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder | snoop_bits, 64); 152173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 153173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 154e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder/* 155e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder * DMA Engine Hardware Control Helpers 156e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder */ 157e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder 158e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyderstatic void dma_init(struct fsldma_chan *chan) 159f79abb627f033c85a6088231f20c85bc4a9bd757Zhang Wei{ 160e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder /* Reset the channel */ 161e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder DMA_OUT(chan, &chan->regs->mr, 0, 32); 162e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder 163e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder switch (chan->feature & FSL_DMA_IP_MASK) { 164e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder case FSL_DMA_IP_85XX: 165e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder /* Set the channel to below modes: 166e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder * EIE - Error interrupt enable 167e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder * EOLNIE - End of links interrupt enable 168e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder * BWC - Bandwidth sharing among channels 169e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder */ 170e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder DMA_OUT(chan, &chan->regs->mr, FSL_DMA_MR_BWC 171f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder | FSL_DMA_MR_EIE | FSL_DMA_MR_EOLNIE, 32); 172e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder break; 173e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder case FSL_DMA_IP_83XX: 174e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder /* Set the channel to below modes: 175e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder * EOTIE - End-of-transfer interrupt enable 176e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder * PRC_RM - PCI read multiple 177e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder */ 178e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder DMA_OUT(chan, &chan->regs->mr, FSL_DMA_MR_EOTIE 179e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder | FSL_DMA_MR_PRC_RM, 32); 180e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder break; 181e8bd84df27c5921a9ac866aef06e044590ac118fIra Snyder } 182f79abb627f033c85a6088231f20c85bc4a9bd757Zhang Wei} 183f79abb627f033c85a6088231f20c85bc4a9bd757Zhang Wei 184a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic int dma_is_idle(struct fsldma_chan *chan) 185173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 186a1c03319018061304be28d131073ac13a5cb86fbIra Snyder u32 sr = get_sr(chan); 187173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei return (!(sr & FSL_DMA_SR_CB)) || (sr & FSL_DMA_SR_CH); 188173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 189173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 190f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder/* 191f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder * Start the DMA controller 192f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder * 193f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder * Preconditions: 194f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder * - the CDAR register must point to the start descriptor 195f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder * - the MRn[CS] bit must be cleared 196f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder */ 197a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void dma_start(struct fsldma_chan *chan) 198173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 199272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder u32 mode; 200272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder 201a1c03319018061304be28d131073ac13a5cb86fbIra Snyder mode = DMA_IN(chan, &chan->regs->mr, 32); 202272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder 203f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder if (chan->feature & FSL_DMA_CHAN_PAUSE_EXT) { 204f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder DMA_OUT(chan, &chan->regs->bcr, 0, 32); 205f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder mode |= FSL_DMA_MR_EMP_EN; 206f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder } else { 207f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder mode &= ~FSL_DMA_MR_EMP_EN; 20843a1a3ed6bf5a1b9ae197b4f5f20033baf19db61Ira Snyder } 209173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 210f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder if (chan->feature & FSL_DMA_CHAN_START_EXT) { 211272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder mode |= FSL_DMA_MR_EMS_EN; 212f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder } else { 213f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder mode &= ~FSL_DMA_MR_EMS_EN; 214272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder mode |= FSL_DMA_MR_CS; 215f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder } 216173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 217a1c03319018061304be28d131073ac13a5cb86fbIra Snyder DMA_OUT(chan, &chan->regs->mr, mode, 32); 218173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 219173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 220a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void dma_halt(struct fsldma_chan *chan) 221173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 222272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder u32 mode; 223900325a6ce33995688b7a680a34e7698f16f4d72Dan Williams int i; 224900325a6ce33995688b7a680a34e7698f16f4d72Dan Williams 225a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder /* read the mode register */ 226a1c03319018061304be28d131073ac13a5cb86fbIra Snyder mode = DMA_IN(chan, &chan->regs->mr, 32); 227272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder 228a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder /* 229a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder * The 85xx controller supports channel abort, which will stop 230a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder * the current transfer. On 83xx, this bit is the transfer error 231a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder * mask bit, which should not be changed. 232a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder */ 233a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) { 234a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder mode |= FSL_DMA_MR_CA; 235a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder DMA_OUT(chan, &chan->regs->mr, mode, 32); 236a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder 237a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder mode &= ~FSL_DMA_MR_CA; 238a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder } 239a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder 240a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder /* stop the DMA controller */ 241a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder mode &= ~(FSL_DMA_MR_CS | FSL_DMA_MR_EMS_EN); 242a1c03319018061304be28d131073ac13a5cb86fbIra Snyder DMA_OUT(chan, &chan->regs->mr, mode, 32); 243173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 244a00ae34ac8bc8a5897d9b6b9b685c39b955b14b9Ira Snyder /* wait for the DMA controller to become idle */ 245900325a6ce33995688b7a680a34e7698f16f4d72Dan Williams for (i = 0; i < 100; i++) { 246a1c03319018061304be28d131073ac13a5cb86fbIra Snyder if (dma_is_idle(chan)) 2479c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder return; 2489c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder 249173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei udelay(10); 250900325a6ce33995688b7a680a34e7698f16f4d72Dan Williams } 251272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder 2529c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder if (!dma_is_idle(chan)) 253b158471ef63bf399165db96e945a828096502d9dIra Snyder chan_err(chan, "DMA halt timeout!\n"); 254173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 255173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 256173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/** 257173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * fsl_chan_set_src_loop_size - Set source address hold transfer size 258a1c03319018061304be28d131073ac13a5cb86fbIra Snyder * @chan : Freescale DMA channel 259173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * @size : Address loop size, 0 for disable loop 260173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * 261173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * The set source address hold transfer size. The source 262173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * address hold or loop transfer size is when the DMA transfer 263173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * data from source address (SA), if the loop size is 4, the DMA will 264173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * read data from SA, SA + 1, SA + 2, SA + 3, then loop back to SA, 265173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * SA + 1 ... and so on. 266173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */ 267a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void fsl_chan_set_src_loop_size(struct fsldma_chan *chan, int size) 268173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 269272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder u32 mode; 270272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder 271a1c03319018061304be28d131073ac13a5cb86fbIra Snyder mode = DMA_IN(chan, &chan->regs->mr, 32); 272272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder 273173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei switch (size) { 274173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei case 0: 275272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder mode &= ~FSL_DMA_MR_SAHE; 276173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei break; 277173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei case 1: 278173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei case 2: 279173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei case 4: 280173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei case 8: 281272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder mode |= FSL_DMA_MR_SAHE | (__ilog2(size) << 14); 282173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei break; 283173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei } 284272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder 285a1c03319018061304be28d131073ac13a5cb86fbIra Snyder DMA_OUT(chan, &chan->regs->mr, mode, 32); 286173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 287173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 288173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/** 289738f5f7e1ae876448cb7d9c82bea258b69386647Ira Snyder * fsl_chan_set_dst_loop_size - Set destination address hold transfer size 290a1c03319018061304be28d131073ac13a5cb86fbIra Snyder * @chan : Freescale DMA channel 291173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * @size : Address loop size, 0 for disable loop 292173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * 293173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * The set destination address hold transfer size. The destination 294173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * address hold or loop transfer size is when the DMA transfer 295173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * data to destination address (TA), if the loop size is 4, the DMA will 296173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * write data to TA, TA + 1, TA + 2, TA + 3, then loop back to TA, 297173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * TA + 1 ... and so on. 298173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */ 299a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void fsl_chan_set_dst_loop_size(struct fsldma_chan *chan, int size) 300173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 301272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder u32 mode; 302272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder 303a1c03319018061304be28d131073ac13a5cb86fbIra Snyder mode = DMA_IN(chan, &chan->regs->mr, 32); 304272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder 305173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei switch (size) { 306173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei case 0: 307272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder mode &= ~FSL_DMA_MR_DAHE; 308173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei break; 309173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei case 1: 310173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei case 2: 311173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei case 4: 312173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei case 8: 313272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder mode |= FSL_DMA_MR_DAHE | (__ilog2(size) << 16); 314173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei break; 315173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei } 316272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder 317a1c03319018061304be28d131073ac13a5cb86fbIra Snyder DMA_OUT(chan, &chan->regs->mr, mode, 32); 318173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 319173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 320173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/** 321e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * fsl_chan_set_request_count - Set DMA Request Count for external control 322a1c03319018061304be28d131073ac13a5cb86fbIra Snyder * @chan : Freescale DMA channel 323e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * @size : Number of bytes to transfer in a single request 324e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * 325e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * The Freescale DMA channel can be controlled by the external signal DREQ#. 326e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * The DMA request count is how many bytes are allowed to transfer before 327e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * pausing the channel, after which a new assertion of DREQ# resumes channel 328e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * operation. 329173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * 330e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * A size of 0 disables external pause control. The maximum size is 1024. 331173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */ 332a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void fsl_chan_set_request_count(struct fsldma_chan *chan, int size) 333173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 334272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder u32 mode; 335272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder 336e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder BUG_ON(size > 1024); 337272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder 338a1c03319018061304be28d131073ac13a5cb86fbIra Snyder mode = DMA_IN(chan, &chan->regs->mr, 32); 339272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder mode |= (__ilog2(size) << 24) & 0x0f000000; 340272ca655090978bdaa2630fc44fb2c03da5576fdIra Snyder 341a1c03319018061304be28d131073ac13a5cb86fbIra Snyder DMA_OUT(chan, &chan->regs->mr, mode, 32); 342e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder} 343173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 344e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder/** 345e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * fsl_chan_toggle_ext_pause - Toggle channel external pause status 346a1c03319018061304be28d131073ac13a5cb86fbIra Snyder * @chan : Freescale DMA channel 347e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * @enable : 0 is disabled, 1 is enabled. 348e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * 349e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * The Freescale DMA channel can be controlled by the external signal DREQ#. 350e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * The DMA Request Count feature should be used in addition to this feature 351e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder * to set the number of bytes to transfer before pausing the channel. 352e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder */ 353a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void fsl_chan_toggle_ext_pause(struct fsldma_chan *chan, int enable) 354e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder{ 355e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder if (enable) 356a1c03319018061304be28d131073ac13a5cb86fbIra Snyder chan->feature |= FSL_DMA_CHAN_PAUSE_EXT; 357e6c7ecb64e08ef346cb7062b4a5421f00bc602bdIra Snyder else 358a1c03319018061304be28d131073ac13a5cb86fbIra Snyder chan->feature &= ~FSL_DMA_CHAN_PAUSE_EXT; 359173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 360173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 361173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/** 362173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * fsl_chan_toggle_ext_start - Toggle channel external start status 363a1c03319018061304be28d131073ac13a5cb86fbIra Snyder * @chan : Freescale DMA channel 364173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * @enable : 0 is disabled, 1 is enabled. 365173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * 366173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * If enable the external start, the channel can be started by an 367173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * external DMA start pin. So the dma_start() does not start the 368173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * transfer immediately. The DMA channel will wait for the 369173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * control pin asserted. 370173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */ 371a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void fsl_chan_toggle_ext_start(struct fsldma_chan *chan, int enable) 372173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 373173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei if (enable) 374a1c03319018061304be28d131073ac13a5cb86fbIra Snyder chan->feature |= FSL_DMA_CHAN_START_EXT; 375173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei else 376a1c03319018061304be28d131073ac13a5cb86fbIra Snyder chan->feature &= ~FSL_DMA_CHAN_START_EXT; 377173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 378173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 37931f4306c83a2daa3e348056b720de511bffe5a9bIra Snyderstatic void append_ld_queue(struct fsldma_chan *chan, struct fsl_desc_sw *desc) 3809c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder{ 3819c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder struct fsl_desc_sw *tail = to_fsl_desc(chan->ld_pending.prev); 3829c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder 3839c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder if (list_empty(&chan->ld_pending)) 3849c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder goto out_splice; 3859c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder 3869c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder /* 3879c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * Add the hardware descriptor to the chain of hardware descriptors 3889c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * that already exists in memory. 3899c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * 3909c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * This will un-set the EOL bit of the existing transaction, and the 3919c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * last link in this transaction will become the EOL descriptor. 3929c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder */ 3939c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder set_desc_next(chan, &tail->hw, desc->async_tx.phys); 3949c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder 3959c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder /* 3969c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * Add the software descriptor and all children to the list 3979c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * of pending transactions 3989c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder */ 3999c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyderout_splice: 4009c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder list_splice_tail_init(&desc->tx_list, &chan->ld_pending); 4019c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder} 4029c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder 403173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Weistatic dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx) 404173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 405a1c03319018061304be28d131073ac13a5cb86fbIra Snyder struct fsldma_chan *chan = to_fsl_chan(tx->chan); 406eda34234578fd822c950fd06b5c5ff7ac08b3001Dan Williams struct fsl_desc_sw *desc = tx_to_fsl_desc(tx); 407eda34234578fd822c950fd06b5c5ff7ac08b3001Dan Williams struct fsl_desc_sw *child; 408173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei unsigned long flags; 409173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei dma_cookie_t cookie; 410173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 411a1c03319018061304be28d131073ac13a5cb86fbIra Snyder spin_lock_irqsave(&chan->desc_lock, flags); 412173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 4139c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder /* 4149c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * assign cookies to all of the software descriptors 4159c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * that make up this transaction 4169c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder */ 417eda34234578fd822c950fd06b5c5ff7ac08b3001Dan Williams list_for_each_entry(child, &desc->tx_list, node) { 418884485e1f12dcd39390f042e772cdbefc9ebb750Russell King - ARM Linux cookie = dma_cookie_assign(&child->async_tx); 419bcfb7465c03a8c62c89da374677df56f6b894d44Ira Snyder } 420bcfb7465c03a8c62c89da374677df56f6b894d44Ira Snyder 4219c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder /* put this transaction onto the tail of the pending queue */ 422a1c03319018061304be28d131073ac13a5cb86fbIra Snyder append_ld_queue(chan, desc); 423173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 424a1c03319018061304be28d131073ac13a5cb86fbIra Snyder spin_unlock_irqrestore(&chan->desc_lock, flags); 425173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 426173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei return cookie; 427173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 428173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 429173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/** 430173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * fsl_dma_alloc_descriptor - Allocate descriptor from channel's DMA pool. 431a1c03319018061304be28d131073ac13a5cb86fbIra Snyder * @chan : Freescale DMA channel 432173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * 433173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * Return - The descriptor allocated. NULL for failed. 434173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */ 43531f4306c83a2daa3e348056b720de511bffe5a9bIra Snyderstatic struct fsl_desc_sw *fsl_dma_alloc_descriptor(struct fsldma_chan *chan) 436173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 4379c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder struct fsl_desc_sw *desc; 438173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei dma_addr_t pdesc; 4399c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder 4409c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder desc = dma_pool_alloc(chan->desc_pool, GFP_ATOMIC, &pdesc); 4419c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder if (!desc) { 442b158471ef63bf399165db96e945a828096502d9dIra Snyder chan_dbg(chan, "out of memory for link descriptor\n"); 4439c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder return NULL; 444173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei } 445173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 4469c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder memset(desc, 0, sizeof(*desc)); 4479c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder INIT_LIST_HEAD(&desc->tx_list); 4489c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); 4499c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder desc->async_tx.tx_submit = fsl_dma_tx_submit; 4509c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder desc->async_tx.phys = pdesc; 4519c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder 4520ab09c36818ca88f65c88f4d8c6d067fbf10578dIra Snyder#ifdef FSL_DMA_LD_DEBUG 4530ab09c36818ca88f65c88f4d8c6d067fbf10578dIra Snyder chan_dbg(chan, "LD %p allocated\n", desc); 4540ab09c36818ca88f65c88f4d8c6d067fbf10578dIra Snyder#endif 4550ab09c36818ca88f65c88f4d8c6d067fbf10578dIra Snyder 4569c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder return desc; 457173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 458173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 459173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/** 460173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * fsl_dma_alloc_chan_resources - Allocate resources for DMA channel. 461a1c03319018061304be28d131073ac13a5cb86fbIra Snyder * @chan : Freescale DMA channel 462173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * 463173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * This function will create a dma pool for descriptor allocation. 464173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * 465173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * Return - The number of descriptors allocated. 466173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */ 467a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic int fsl_dma_alloc_chan_resources(struct dma_chan *dchan) 468173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 469a1c03319018061304be28d131073ac13a5cb86fbIra Snyder struct fsldma_chan *chan = to_fsl_chan(dchan); 47077cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi 47177cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi /* Has this channel already been allocated? */ 472a1c03319018061304be28d131073ac13a5cb86fbIra Snyder if (chan->desc_pool) 47377cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi return 1; 474173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 4759c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder /* 4769c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * We need the descriptor to be aligned to 32bytes 477173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * for meeting FSL DMA specification requirement. 478173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */ 479b158471ef63bf399165db96e945a828096502d9dIra Snyder chan->desc_pool = dma_pool_create(chan->name, chan->dev, 4809c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder sizeof(struct fsl_desc_sw), 4819c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder __alignof__(struct fsl_desc_sw), 0); 482a1c03319018061304be28d131073ac13a5cb86fbIra Snyder if (!chan->desc_pool) { 483b158471ef63bf399165db96e945a828096502d9dIra Snyder chan_err(chan, "unable to allocate descriptor pool\n"); 4849c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder return -ENOMEM; 485173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei } 486173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 4879c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder /* there is at least one descriptor free to be allocated */ 488173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei return 1; 489173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 490173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 491173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/** 4929c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * fsldma_free_desc_list - Free all descriptors in a queue 4939c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * @chan: Freescae DMA channel 4949c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * @list: the list to free 4959c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * 4969c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * LOCKING: must hold chan->desc_lock 4979c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder */ 4989c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyderstatic void fsldma_free_desc_list(struct fsldma_chan *chan, 4999c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder struct list_head *list) 5009c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder{ 5019c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder struct fsl_desc_sw *desc, *_desc; 5029c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder 5039c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder list_for_each_entry_safe(desc, _desc, list, node) { 5049c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder list_del(&desc->node); 5050ab09c36818ca88f65c88f4d8c6d067fbf10578dIra Snyder#ifdef FSL_DMA_LD_DEBUG 5060ab09c36818ca88f65c88f4d8c6d067fbf10578dIra Snyder chan_dbg(chan, "LD %p free\n", desc); 5070ab09c36818ca88f65c88f4d8c6d067fbf10578dIra Snyder#endif 5089c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys); 5099c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder } 5109c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder} 5119c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder 5129c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyderstatic void fsldma_free_desc_list_reverse(struct fsldma_chan *chan, 5139c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder struct list_head *list) 5149c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder{ 5159c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder struct fsl_desc_sw *desc, *_desc; 5169c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder 5179c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder list_for_each_entry_safe_reverse(desc, _desc, list, node) { 5189c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder list_del(&desc->node); 5190ab09c36818ca88f65c88f4d8c6d067fbf10578dIra Snyder#ifdef FSL_DMA_LD_DEBUG 5200ab09c36818ca88f65c88f4d8c6d067fbf10578dIra Snyder chan_dbg(chan, "LD %p free\n", desc); 5210ab09c36818ca88f65c88f4d8c6d067fbf10578dIra Snyder#endif 5229c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys); 5239c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder } 5249c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder} 5259c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder 5269c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder/** 527173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * fsl_dma_free_chan_resources - Free all resources of the channel. 528a1c03319018061304be28d131073ac13a5cb86fbIra Snyder * @chan : Freescale DMA channel 529173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */ 530a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void fsl_dma_free_chan_resources(struct dma_chan *dchan) 531173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 532a1c03319018061304be28d131073ac13a5cb86fbIra Snyder struct fsldma_chan *chan = to_fsl_chan(dchan); 533173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei unsigned long flags; 534173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 535b158471ef63bf399165db96e945a828096502d9dIra Snyder chan_dbg(chan, "free all channel resources\n"); 536a1c03319018061304be28d131073ac13a5cb86fbIra Snyder spin_lock_irqsave(&chan->desc_lock, flags); 5379c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder fsldma_free_desc_list(chan, &chan->ld_pending); 5389c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder fsldma_free_desc_list(chan, &chan->ld_running); 539a1c03319018061304be28d131073ac13a5cb86fbIra Snyder spin_unlock_irqrestore(&chan->desc_lock, flags); 54077cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi 5419c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder dma_pool_destroy(chan->desc_pool); 542a1c03319018061304be28d131073ac13a5cb86fbIra Snyder chan->desc_pool = NULL; 543173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 544173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 5452187c269ad29510f1d65ec684133d1d3426d0eedZhang Weistatic struct dma_async_tx_descriptor * 546a1c03319018061304be28d131073ac13a5cb86fbIra Snyderfsl_dma_prep_interrupt(struct dma_chan *dchan, unsigned long flags) 5472187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei{ 548a1c03319018061304be28d131073ac13a5cb86fbIra Snyder struct fsldma_chan *chan; 5492187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei struct fsl_desc_sw *new; 5502187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei 551a1c03319018061304be28d131073ac13a5cb86fbIra Snyder if (!dchan) 5522187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei return NULL; 5532187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei 554a1c03319018061304be28d131073ac13a5cb86fbIra Snyder chan = to_fsl_chan(dchan); 5552187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei 556a1c03319018061304be28d131073ac13a5cb86fbIra Snyder new = fsl_dma_alloc_descriptor(chan); 5572187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei if (!new) { 558b158471ef63bf399165db96e945a828096502d9dIra Snyder chan_err(chan, "%s\n", msg_ld_oom); 5592187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei return NULL; 5602187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei } 5612187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei 5622187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei new->async_tx.cookie = -EBUSY; 563636bdeaa1243327501edfd2a597ed7443eb4239aDan Williams new->async_tx.flags = flags; 5642187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei 565f79abb627f033c85a6088231f20c85bc4a9bd757Zhang Wei /* Insert the link descriptor to the LD ring */ 566eda34234578fd822c950fd06b5c5ff7ac08b3001Dan Williams list_add_tail(&new->node, &new->tx_list); 567f79abb627f033c85a6088231f20c85bc4a9bd757Zhang Wei 56831f4306c83a2daa3e348056b720de511bffe5a9bIra Snyder /* Set End-of-link to the last link descriptor of new list */ 569a1c03319018061304be28d131073ac13a5cb86fbIra Snyder set_ld_eol(chan, new); 5702187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei 5712187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei return &new->async_tx; 5722187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei} 5732187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei 57431f4306c83a2daa3e348056b720de511bffe5a9bIra Snyderstatic struct dma_async_tx_descriptor * 57531f4306c83a2daa3e348056b720de511bffe5a9bIra Snyderfsl_dma_prep_memcpy(struct dma_chan *dchan, 57631f4306c83a2daa3e348056b720de511bffe5a9bIra Snyder dma_addr_t dma_dst, dma_addr_t dma_src, 577173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei size_t len, unsigned long flags) 578173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 579a1c03319018061304be28d131073ac13a5cb86fbIra Snyder struct fsldma_chan *chan; 580173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei struct fsl_desc_sw *first = NULL, *prev = NULL, *new; 581173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei size_t copy; 582173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 583a1c03319018061304be28d131073ac13a5cb86fbIra Snyder if (!dchan) 584173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei return NULL; 585173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 586173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei if (!len) 587173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei return NULL; 588173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 589a1c03319018061304be28d131073ac13a5cb86fbIra Snyder chan = to_fsl_chan(dchan); 590173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 591173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei do { 592173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 593173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei /* Allocate the link descriptor from DMA pool */ 594a1c03319018061304be28d131073ac13a5cb86fbIra Snyder new = fsl_dma_alloc_descriptor(chan); 595173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei if (!new) { 596b158471ef63bf399165db96e945a828096502d9dIra Snyder chan_err(chan, "%s\n", msg_ld_oom); 5972e077f8e8337e52eef3c39c24c31e103b11a0326Ira Snyder goto fail; 598173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei } 599173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 60056822843ff99c88c778a614851328fcbb1503d10Zhang Wei copy = min(len, (size_t)FSL_DMA_BCR_MAX_CNT); 601173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 602a1c03319018061304be28d131073ac13a5cb86fbIra Snyder set_desc_cnt(chan, &new->hw, copy); 603a1c03319018061304be28d131073ac13a5cb86fbIra Snyder set_desc_src(chan, &new->hw, dma_src); 604a1c03319018061304be28d131073ac13a5cb86fbIra Snyder set_desc_dst(chan, &new->hw, dma_dst); 605173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 606173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei if (!first) 607173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei first = new; 608173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei else 609a1c03319018061304be28d131073ac13a5cb86fbIra Snyder set_desc_next(chan, &prev->hw, new->async_tx.phys); 610173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 611173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei new->async_tx.cookie = 0; 612636bdeaa1243327501edfd2a597ed7443eb4239aDan Williams async_tx_ack(&new->async_tx); 613173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 614173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei prev = new; 615173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei len -= copy; 616173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei dma_src += copy; 617738f5f7e1ae876448cb7d9c82bea258b69386647Ira Snyder dma_dst += copy; 618173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 619173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei /* Insert the link descriptor to the LD ring */ 620eda34234578fd822c950fd06b5c5ff7ac08b3001Dan Williams list_add_tail(&new->node, &first->tx_list); 621173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei } while (len); 622173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 623636bdeaa1243327501edfd2a597ed7443eb4239aDan Williams new->async_tx.flags = flags; /* client is in control of this ack */ 624173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei new->async_tx.cookie = -EBUSY; 625173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 62631f4306c83a2daa3e348056b720de511bffe5a9bIra Snyder /* Set End-of-link to the last link descriptor of new list */ 627a1c03319018061304be28d131073ac13a5cb86fbIra Snyder set_ld_eol(chan, new); 628173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 6292e077f8e8337e52eef3c39c24c31e103b11a0326Ira Snyder return &first->async_tx; 6302e077f8e8337e52eef3c39c24c31e103b11a0326Ira Snyder 6312e077f8e8337e52eef3c39c24c31e103b11a0326Ira Snyderfail: 6322e077f8e8337e52eef3c39c24c31e103b11a0326Ira Snyder if (!first) 6332e077f8e8337e52eef3c39c24c31e103b11a0326Ira Snyder return NULL; 6342e077f8e8337e52eef3c39c24c31e103b11a0326Ira Snyder 6359c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder fsldma_free_desc_list_reverse(chan, &first->tx_list); 6362e077f8e8337e52eef3c39c24c31e103b11a0326Ira Snyder return NULL; 637173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 638173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 639c14330417ef2050f4bf38ac20e125785fea14351Ira Snyderstatic struct dma_async_tx_descriptor *fsl_dma_prep_sg(struct dma_chan *dchan, 640c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder struct scatterlist *dst_sg, unsigned int dst_nents, 641c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder struct scatterlist *src_sg, unsigned int src_nents, 642c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder unsigned long flags) 643c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder{ 644c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder struct fsl_desc_sw *first = NULL, *prev = NULL, *new = NULL; 645c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder struct fsldma_chan *chan = to_fsl_chan(dchan); 646c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder size_t dst_avail, src_avail; 647c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder dma_addr_t dst, src; 648c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder size_t len; 649c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 650c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder /* basic sanity checks */ 651c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder if (dst_nents == 0 || src_nents == 0) 652c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder return NULL; 653c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 654c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder if (dst_sg == NULL || src_sg == NULL) 655c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder return NULL; 656c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 657c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder /* 658c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder * TODO: should we check that both scatterlists have the same 659c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder * TODO: number of bytes in total? Is that really an error? 660c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder */ 661c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 662c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder /* get prepared for the loop */ 663c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder dst_avail = sg_dma_len(dst_sg); 664c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder src_avail = sg_dma_len(src_sg); 665c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 666c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder /* run until we are out of scatterlist entries */ 667c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder while (true) { 668c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 669c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder /* create the largest transaction possible */ 670c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder len = min_t(size_t, src_avail, dst_avail); 671c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder len = min_t(size_t, len, FSL_DMA_BCR_MAX_CNT); 672c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder if (len == 0) 673c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder goto fetch; 674c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 675c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder dst = sg_dma_address(dst_sg) + sg_dma_len(dst_sg) - dst_avail; 676c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder src = sg_dma_address(src_sg) + sg_dma_len(src_sg) - src_avail; 677c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 678c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder /* allocate and populate the descriptor */ 679c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder new = fsl_dma_alloc_descriptor(chan); 680c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder if (!new) { 681b158471ef63bf399165db96e945a828096502d9dIra Snyder chan_err(chan, "%s\n", msg_ld_oom); 682c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder goto fail; 683c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder } 684c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 685c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder set_desc_cnt(chan, &new->hw, len); 686c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder set_desc_src(chan, &new->hw, src); 687c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder set_desc_dst(chan, &new->hw, dst); 688c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 689c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder if (!first) 690c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder first = new; 691c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder else 692c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder set_desc_next(chan, &prev->hw, new->async_tx.phys); 693c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 694c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder new->async_tx.cookie = 0; 695c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder async_tx_ack(&new->async_tx); 696c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder prev = new; 697c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 698c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder /* Insert the link descriptor to the LD ring */ 699c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder list_add_tail(&new->node, &first->tx_list); 700c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 701c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder /* update metadata */ 702c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder dst_avail -= len; 703c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder src_avail -= len; 704c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 705c14330417ef2050f4bf38ac20e125785fea14351Ira Snyderfetch: 706c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder /* fetch the next dst scatterlist entry */ 707c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder if (dst_avail == 0) { 708c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 709c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder /* no more entries: we're done */ 710c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder if (dst_nents == 0) 711c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder break; 712c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 713c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder /* fetch the next entry: if there are no more: done */ 714c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder dst_sg = sg_next(dst_sg); 715c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder if (dst_sg == NULL) 716c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder break; 717c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 718c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder dst_nents--; 719c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder dst_avail = sg_dma_len(dst_sg); 720c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder } 721c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 722c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder /* fetch the next src scatterlist entry */ 723c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder if (src_avail == 0) { 724c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 725c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder /* no more entries: we're done */ 726c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder if (src_nents == 0) 727c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder break; 728c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 729c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder /* fetch the next entry: if there are no more: done */ 730c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder src_sg = sg_next(src_sg); 731c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder if (src_sg == NULL) 732c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder break; 733c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 734c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder src_nents--; 735c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder src_avail = sg_dma_len(src_sg); 736c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder } 737c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder } 738c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 739c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder new->async_tx.flags = flags; /* client is in control of this ack */ 740c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder new->async_tx.cookie = -EBUSY; 741c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 742c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder /* Set End-of-link to the last link descriptor of new list */ 743c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder set_ld_eol(chan, new); 744c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 745c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder return &first->async_tx; 746c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 747c14330417ef2050f4bf38ac20e125785fea14351Ira Snyderfail: 748c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder if (!first) 749c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder return NULL; 750c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 751c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder fsldma_free_desc_list_reverse(chan, &first->tx_list); 752c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder return NULL; 753c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder} 754c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder 755173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/** 756bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder * fsl_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction 757bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder * @chan: DMA channel 758bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder * @sgl: scatterlist to transfer to/from 759bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder * @sg_len: number of entries in @scatterlist 760bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder * @direction: DMA direction 761bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder * @flags: DMAEngine flags 762185ecb5f4fd43911c35956d4cc7d94a1da30417fAlexandre Bounine * @context: transaction context (ignored) 763bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder * 764bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder * Prepare a set of descriptors for a DMA_SLAVE transaction. Following the 765bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder * DMA_SLAVE API, this gets the device-specific information from the 766bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder * chan->private variable. 767bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder */ 768bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyderstatic struct dma_async_tx_descriptor *fsl_dma_prep_slave_sg( 769a1c03319018061304be28d131073ac13a5cb86fbIra Snyder struct dma_chan *dchan, struct scatterlist *sgl, unsigned int sg_len, 770185ecb5f4fd43911c35956d4cc7d94a1da30417fAlexandre Bounine enum dma_transfer_direction direction, unsigned long flags, 771185ecb5f4fd43911c35956d4cc7d94a1da30417fAlexandre Bounine void *context) 772bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder{ 773bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder /* 774968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder * This operation is not supported on the Freescale DMA controller 775bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder * 776968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder * However, we need to provide the function pointer to allow the 777968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder * device_control() method to work. 778bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder */ 779bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder return NULL; 780bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder} 781bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder 782c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleijstatic int fsl_dma_device_control(struct dma_chan *dchan, 783058276303dbc4ed089c1f7dad0871810b1f5ddf1Linus Walleij enum dma_ctrl_cmd cmd, unsigned long arg) 784bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder{ 785968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder struct dma_slave_config *config; 786a1c03319018061304be28d131073ac13a5cb86fbIra Snyder struct fsldma_chan *chan; 787bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder unsigned long flags; 788968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder int size; 789c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij 790a1c03319018061304be28d131073ac13a5cb86fbIra Snyder if (!dchan) 791c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij return -EINVAL; 792bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder 793a1c03319018061304be28d131073ac13a5cb86fbIra Snyder chan = to_fsl_chan(dchan); 794bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder 795968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder switch (cmd) { 796968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder case DMA_TERMINATE_ALL: 797f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder spin_lock_irqsave(&chan->desc_lock, flags); 798f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder 799968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder /* Halt the DMA engine */ 800968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder dma_halt(chan); 801bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder 802968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder /* Remove and free all of the descriptors in the LD queue */ 803968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder fsldma_free_desc_list(chan, &chan->ld_pending); 804968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder fsldma_free_desc_list(chan, &chan->ld_running); 805f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder chan->idle = true; 806bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder 807968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder spin_unlock_irqrestore(&chan->desc_lock, flags); 808968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder return 0; 809968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder 810968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder case DMA_SLAVE_CONFIG: 811968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder config = (struct dma_slave_config *)arg; 812968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder 813968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder /* make sure the channel supports setting burst size */ 814968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder if (!chan->set_request_count) 815968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder return -ENXIO; 816968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder 817968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder /* we set the controller burst size depending on direction */ 818db8196df4bb6f117caa163aa73b0f16fd62290bdVinod Koul if (config->direction == DMA_MEM_TO_DEV) 819968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder size = config->dst_addr_width * config->dst_maxburst; 820968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder else 821968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder size = config->src_addr_width * config->src_maxburst; 822968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder 823968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder chan->set_request_count(chan, size); 824968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder return 0; 825968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder 826968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder case FSLDMA_EXTERNAL_START: 827968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder 828968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder /* make sure the channel supports external start */ 829968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder if (!chan->toggle_ext_start) 830968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder return -ENXIO; 831968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder 832968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder chan->toggle_ext_start(chan, arg); 833968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder return 0; 834968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder 835968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder default: 836968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder return -ENXIO; 837968f19ae802fdc6b6b6b5af6fe79cf23d281be0fIra Snyder } 838c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij 839c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij return 0; 840bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder} 841bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder 842bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder/** 8439c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder * fsldma_cleanup_descriptor - cleanup and free a single link descriptor 8449c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * @chan: Freescale DMA channel 8459c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder * @desc: descriptor to cleanup and free 846173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * 8479c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder * This function is used on a descriptor which has been executed by the DMA 8489c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder * controller. It will run any callbacks, submit any dependencies, and then 8499c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder * free the descriptor. 850173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */ 8519c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyderstatic void fsldma_cleanup_descriptor(struct fsldma_chan *chan, 8529c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder struct fsl_desc_sw *desc) 853173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 8549c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder struct dma_async_tx_descriptor *txd = &desc->async_tx; 8559c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder struct device *dev = chan->common.device->dev; 8569c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder dma_addr_t src = get_desc_src(chan, desc); 8579c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder dma_addr_t dst = get_desc_dst(chan, desc); 8589c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder u32 len = get_desc_cnt(chan, desc); 8599c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder 8609c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder /* Run the link descriptor callback function */ 8619c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder if (txd->callback) { 8629c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder#ifdef FSL_DMA_LD_DEBUG 8639c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder chan_dbg(chan, "LD %p callback\n", desc); 8649c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder#endif 8659c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder txd->callback(txd->callback_param); 8669c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder } 867173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 8689c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder /* Run any dependencies */ 8699c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder dma_run_dependencies(txd); 870173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 8719c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder /* Unmap the dst buffer, if requested */ 8729c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) { 8739c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE) 8749c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder dma_unmap_single(dev, dst, len, DMA_FROM_DEVICE); 8759c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder else 8769c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder dma_unmap_page(dev, dst, len, DMA_FROM_DEVICE); 8779c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder } 8789c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder 8799c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder /* Unmap the src buffer, if requested */ 8809c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) { 8819c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE) 8829c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder dma_unmap_single(dev, src, len, DMA_TO_DEVICE); 8839c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder else 8849c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder dma_unmap_page(dev, src, len, DMA_TO_DEVICE); 885173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei } 8869c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder 8879c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder#ifdef FSL_DMA_LD_DEBUG 8889c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder chan_dbg(chan, "LD %p free\n", desc); 8899c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder#endif 8909c4d1e7bdeb1ed4dc0c3341d40662a6fbc5f2dc2Ira Snyder dma_pool_free(chan->desc_pool, desc, txd->phys); 891173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 892173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 893173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/** 8949c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * fsl_chan_xfer_ld_queue - transfer any pending transactions 895a1c03319018061304be28d131073ac13a5cb86fbIra Snyder * @chan : Freescale DMA channel 8969c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * 897f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder * HARDWARE STATE: idle 898dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder * LOCKING: must hold chan->desc_lock 899173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */ 900a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void fsl_chan_xfer_ld_queue(struct fsldma_chan *chan) 901173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 9029c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder struct fsl_desc_sw *desc; 903138ef0185177a6d221d24b6aa8f12d867fbbef90Ira Snyder 9049c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder /* 9059c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * If the list of pending descriptors is empty, then we 9069c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * don't need to do any work at all 9079c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder */ 9089c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder if (list_empty(&chan->ld_pending)) { 909b158471ef63bf399165db96e945a828096502d9dIra Snyder chan_dbg(chan, "no pending LDs\n"); 910dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder return; 9119c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder } 912173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 9139c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder /* 914f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder * The DMA controller is not idle, which means that the interrupt 915f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder * handler will start any queued transactions when it runs after 916f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder * this transaction finishes 9179c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder */ 918f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder if (!chan->idle) { 919b158471ef63bf399165db96e945a828096502d9dIra Snyder chan_dbg(chan, "DMA controller still busy\n"); 920dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder return; 9219c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder } 9229c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder 9239c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder /* 9249c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * If there are some link descriptors which have not been 9259c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * transferred, we need to start the controller 926173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */ 927173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 9289c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder /* 9299c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * Move all elements from the queue of pending transactions 9309c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * onto the list of running transactions 9319c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder */ 932f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder chan_dbg(chan, "idle, starting controller\n"); 9339c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder desc = list_first_entry(&chan->ld_pending, struct fsl_desc_sw, node); 9349c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder list_splice_tail_init(&chan->ld_pending, &chan->ld_running); 9359c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder 9369c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder /* 937f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder * The 85xx DMA controller doesn't clear the channel start bit 938f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder * automatically at the end of a transfer. Therefore we must clear 939f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder * it in software before starting the transfer. 940f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder */ 941f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) { 942f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder u32 mode; 943f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder 944f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder mode = DMA_IN(chan, &chan->regs->mr, 32); 945f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder mode &= ~FSL_DMA_MR_CS; 946f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder DMA_OUT(chan, &chan->regs->mr, mode, 32); 947f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder } 948f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder 949f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder /* 9509c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * Program the descriptor's address into the DMA controller, 9519c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * then start the DMA transaction 9529c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder */ 9539c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder set_cdar(chan, desc->async_tx.phys); 954f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder get_cdar(chan); 955138ef0185177a6d221d24b6aa8f12d867fbbef90Ira Snyder 9569c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder dma_start(chan); 957f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder chan->idle = false; 958173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 959173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 960173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/** 961173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * fsl_dma_memcpy_issue_pending - Issue the DMA start command 962a1c03319018061304be28d131073ac13a5cb86fbIra Snyder * @chan : Freescale DMA channel 963173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */ 964a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void fsl_dma_memcpy_issue_pending(struct dma_chan *dchan) 965173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 966a1c03319018061304be28d131073ac13a5cb86fbIra Snyder struct fsldma_chan *chan = to_fsl_chan(dchan); 967dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder unsigned long flags; 968dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder 969dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder spin_lock_irqsave(&chan->desc_lock, flags); 970a1c03319018061304be28d131073ac13a5cb86fbIra Snyder fsl_chan_xfer_ld_queue(chan); 971dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder spin_unlock_irqrestore(&chan->desc_lock, flags); 972173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 973173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 974173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei/** 9750793448187643b50af89d36b08470baf45a3cab4Linus Walleij * fsl_tx_status - Determine the DMA status 976a1c03319018061304be28d131073ac13a5cb86fbIra Snyder * @chan : Freescale DMA channel 977173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */ 9780793448187643b50af89d36b08470baf45a3cab4Linus Walleijstatic enum dma_status fsl_tx_status(struct dma_chan *dchan, 979173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei dma_cookie_t cookie, 9800793448187643b50af89d36b08470baf45a3cab4Linus Walleij struct dma_tx_state *txstate) 981173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 982a1c03319018061304be28d131073ac13a5cb86fbIra Snyder struct fsldma_chan *chan = to_fsl_chan(dchan); 98396a2af41c78b1fbb1f567a3486bdc63f7b31c5fdRussell King - ARM Linux enum dma_status ret; 984f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder unsigned long flags; 985173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 986f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder spin_lock_irqsave(&chan->desc_lock, flags); 98796a2af41c78b1fbb1f567a3486bdc63f7b31c5fdRussell King - ARM Linux ret = dma_cookie_status(dchan, cookie, txstate); 988f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder spin_unlock_irqrestore(&chan->desc_lock, flags); 989173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 99096a2af41c78b1fbb1f567a3486bdc63f7b31c5fdRussell King - ARM Linux return ret; 991173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 992173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 993d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder/*----------------------------------------------------------------------------*/ 994d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder/* Interrupt Handling */ 995d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder/*----------------------------------------------------------------------------*/ 996d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder 997e7a29151de1bd52081f27f149b68074fac0323beIra Snyderstatic irqreturn_t fsldma_chan_irq(int irq, void *data) 998173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 999a1c03319018061304be28d131073ac13a5cb86fbIra Snyder struct fsldma_chan *chan = data; 1000a1c03319018061304be28d131073ac13a5cb86fbIra Snyder u32 stat; 1001173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 10029c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder /* save and clear the status register */ 1003a1c03319018061304be28d131073ac13a5cb86fbIra Snyder stat = get_sr(chan); 10049c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder set_sr(chan, stat); 1005b158471ef63bf399165db96e945a828096502d9dIra Snyder chan_dbg(chan, "irq: stat = 0x%x\n", stat); 1006173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1007f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder /* check that this was really our device */ 1008173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei stat &= ~(FSL_DMA_SR_CB | FSL_DMA_SR_CH); 1009173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei if (!stat) 1010173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei return IRQ_NONE; 1011173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1012173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei if (stat & FSL_DMA_SR_TE) 1013b158471ef63bf399165db96e945a828096502d9dIra Snyder chan_err(chan, "Transfer Error!\n"); 1014173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 10159c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder /* 10169c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * Programming Error 1017f79abb627f033c85a6088231f20c85bc4a9bd757Zhang Wei * The DMA_INTERRUPT async_tx is a NULL transfer, which will 1018f79abb627f033c85a6088231f20c85bc4a9bd757Zhang Wei * triger a PE interrupt. 1019f79abb627f033c85a6088231f20c85bc4a9bd757Zhang Wei */ 1020f79abb627f033c85a6088231f20c85bc4a9bd757Zhang Wei if (stat & FSL_DMA_SR_PE) { 1021b158471ef63bf399165db96e945a828096502d9dIra Snyder chan_dbg(chan, "irq: Programming Error INT\n"); 1022f79abb627f033c85a6088231f20c85bc4a9bd757Zhang Wei stat &= ~FSL_DMA_SR_PE; 1023f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder if (get_bcr(chan) != 0) 1024f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder chan_err(chan, "Programming Error!\n"); 10251c62979ed29a8e2bf9fbe1db101c81a0089676f8Zhang Wei } 10261c62979ed29a8e2bf9fbe1db101c81a0089676f8Zhang Wei 10279c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder /* 10289c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * For MPC8349, EOCDI event need to update cookie 10291c62979ed29a8e2bf9fbe1db101c81a0089676f8Zhang Wei * and start the next transfer if it exist. 10301c62979ed29a8e2bf9fbe1db101c81a0089676f8Zhang Wei */ 10311c62979ed29a8e2bf9fbe1db101c81a0089676f8Zhang Wei if (stat & FSL_DMA_SR_EOCDI) { 1032b158471ef63bf399165db96e945a828096502d9dIra Snyder chan_dbg(chan, "irq: End-of-Chain link INT\n"); 10331c62979ed29a8e2bf9fbe1db101c81a0089676f8Zhang Wei stat &= ~FSL_DMA_SR_EOCDI; 1034173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei } 1035173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 10369c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder /* 10379c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder * If it current transfer is the end-of-transfer, 1038173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * we should clear the Channel Start bit for 1039173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei * prepare next transfer. 1040173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */ 10411c62979ed29a8e2bf9fbe1db101c81a0089676f8Zhang Wei if (stat & FSL_DMA_SR_EOLNI) { 1042b158471ef63bf399165db96e945a828096502d9dIra Snyder chan_dbg(chan, "irq: End-of-link INT\n"); 1043173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei stat &= ~FSL_DMA_SR_EOLNI; 1044173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei } 1045173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1046f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder /* check that the DMA controller is really idle */ 1047f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder if (!dma_is_idle(chan)) 1048f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder chan_err(chan, "irq: controller not idle!\n"); 1049f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder 1050f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder /* check that we handled all of the bits */ 1051173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei if (stat) 1052f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder chan_err(chan, "irq: unhandled sr 0x%08x\n", stat); 1053173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1054f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder /* 1055f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder * Schedule the tasklet to handle all cleanup of the current 1056f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder * transaction. It will start a new transaction if there is 1057f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder * one pending. 1058f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder */ 1059a1c03319018061304be28d131073ac13a5cb86fbIra Snyder tasklet_schedule(&chan->tasklet); 1060f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder chan_dbg(chan, "irq: Exit\n"); 1061173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei return IRQ_HANDLED; 1062173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 1063173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1064d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyderstatic void dma_do_tasklet(unsigned long data) 1065d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder{ 1066a1c03319018061304be28d131073ac13a5cb86fbIra Snyder struct fsldma_chan *chan = (struct fsldma_chan *)data; 1067dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder struct fsl_desc_sw *desc, *_desc; 1068dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder LIST_HEAD(ld_cleanup); 1069f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder unsigned long flags; 1070f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder 1071f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder chan_dbg(chan, "tasklet entry\n"); 1072f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder 1073f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder spin_lock_irqsave(&chan->desc_lock, flags); 1074dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder 1075dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder /* update the cookie if we have some descriptors to cleanup */ 1076dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder if (!list_empty(&chan->ld_running)) { 1077dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder dma_cookie_t cookie; 1078dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder 1079dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder desc = to_fsl_desc(chan->ld_running.prev); 1080dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder cookie = desc->async_tx.cookie; 1081f7fbce07c6ce26a25b4e0cb5f241c361fde87901Russell King - ARM Linux dma_cookie_complete(&desc->async_tx); 1082dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder 1083dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder chan_dbg(chan, "completed_cookie=%d\n", cookie); 1084dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder } 1085dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder 1086dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder /* 1087dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder * move the descriptors to a temporary list so we can drop the lock 1088dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder * during the entire cleanup operation 1089dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder */ 1090dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder list_splice_tail_init(&chan->ld_running, &ld_cleanup); 1091dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder 1092dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder /* the hardware is now idle and ready for more */ 1093f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder chan->idle = true; 1094f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder 1095dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder /* 1096dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder * Start any pending transactions automatically 1097dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder * 1098dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder * In the ideal case, we keep the DMA controller busy while we go 1099dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder * ahead and free the descriptors below. 1100dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder */ 1101f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder fsl_chan_xfer_ld_queue(chan); 1102dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder spin_unlock_irqrestore(&chan->desc_lock, flags); 1103dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder 1104dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder /* Run the callback for each descriptor, in order */ 1105dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder list_for_each_entry_safe(desc, _desc, &ld_cleanup, node) { 1106dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder 1107dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder /* Remove from the list of transactions */ 1108dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder list_del(&desc->node); 1109dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder 1110dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder /* Run all cleanup for this descriptor */ 1111dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder fsldma_cleanup_descriptor(chan, desc); 1112dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder } 1113dc8d4091575ba81e886ebcdfd1e559c981f82f86Ira Snyder 1114f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder chan_dbg(chan, "tasklet exit\n"); 1115d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder} 1116d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder 1117d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyderstatic irqreturn_t fsldma_ctrl_irq(int irq, void *data) 1118173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 1119a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder struct fsldma_device *fdev = data; 1120d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder struct fsldma_chan *chan; 1121d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder unsigned int handled = 0; 1122d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder u32 gsr, mask; 1123d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder int i; 1124173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1125e7a29151de1bd52081f27f149b68074fac0323beIra Snyder gsr = (fdev->feature & FSL_DMA_BIG_ENDIAN) ? in_be32(fdev->regs) 1126d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder : in_le32(fdev->regs); 1127d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder mask = 0xff000000; 1128d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder dev_dbg(fdev->dev, "IRQ: gsr 0x%.8x\n", gsr); 1129173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1130d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { 1131d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder chan = fdev->chan[i]; 1132d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder if (!chan) 1133d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder continue; 1134d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder 1135d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder if (gsr & mask) { 1136d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder dev_dbg(fdev->dev, "IRQ: chan %d\n", chan->id); 1137d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder fsldma_chan_irq(irq, chan); 1138d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder handled++; 1139d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder } 1140d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder 1141d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder gsr &= ~mask; 1142d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder mask >>= 8; 1143d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder } 1144d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder 1145d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder return IRQ_RETVAL(handled); 1146173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 1147173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1148d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyderstatic void fsldma_free_irqs(struct fsldma_device *fdev) 1149173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 1150d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder struct fsldma_chan *chan; 1151d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder int i; 1152d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder 1153d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder if (fdev->irq != NO_IRQ) { 1154d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder dev_dbg(fdev->dev, "free per-controller IRQ\n"); 1155d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder free_irq(fdev->irq, fdev); 1156d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder return; 1157d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder } 1158d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder 1159d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { 1160d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder chan = fdev->chan[i]; 1161d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder if (chan && chan->irq != NO_IRQ) { 1162b158471ef63bf399165db96e945a828096502d9dIra Snyder chan_dbg(chan, "free per-channel IRQ\n"); 1163d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder free_irq(chan->irq, chan); 1164d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder } 1165d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder } 1166d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder} 1167d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder 1168d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyderstatic int fsldma_request_irqs(struct fsldma_device *fdev) 1169d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder{ 1170d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder struct fsldma_chan *chan; 1171d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder int ret; 1172d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder int i; 1173d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder 1174d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder /* if we have a per-controller IRQ, use that */ 1175d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder if (fdev->irq != NO_IRQ) { 1176d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder dev_dbg(fdev->dev, "request per-controller IRQ\n"); 1177d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder ret = request_irq(fdev->irq, fsldma_ctrl_irq, IRQF_SHARED, 1178d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder "fsldma-controller", fdev); 1179d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder return ret; 1180d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder } 1181d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder 1182d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder /* no per-controller IRQ, use the per-channel IRQs */ 1183d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { 1184d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder chan = fdev->chan[i]; 1185d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder if (!chan) 1186d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder continue; 1187d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder 1188d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder if (chan->irq == NO_IRQ) { 1189b158471ef63bf399165db96e945a828096502d9dIra Snyder chan_err(chan, "interrupts property missing in device tree\n"); 1190d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder ret = -ENODEV; 1191d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder goto out_unwind; 1192d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder } 1193d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder 1194b158471ef63bf399165db96e945a828096502d9dIra Snyder chan_dbg(chan, "request per-channel IRQ\n"); 1195d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder ret = request_irq(chan->irq, fsldma_chan_irq, IRQF_SHARED, 1196d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder "fsldma-chan", chan); 1197d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder if (ret) { 1198b158471ef63bf399165db96e945a828096502d9dIra Snyder chan_err(chan, "unable to request per-channel IRQ\n"); 1199d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder goto out_unwind; 1200d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder } 1201d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder } 1202d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder 1203d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder return 0; 1204d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder 1205d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyderout_unwind: 1206d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder for (/* none */; i >= 0; i--) { 1207d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder chan = fdev->chan[i]; 1208d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder if (!chan) 1209d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder continue; 1210d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder 1211d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder if (chan->irq == NO_IRQ) 1212d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder continue; 1213d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder 1214d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder free_irq(chan->irq, chan); 1215d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder } 1216d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder 1217d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder return ret; 1218173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 1219173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1220a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder/*----------------------------------------------------------------------------*/ 1221a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder/* OpenFirmware Subsystem */ 1222a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder/*----------------------------------------------------------------------------*/ 1223a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder 1224a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyderstatic int __devinit fsl_dma_chan_probe(struct fsldma_device *fdev, 122577cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi struct device_node *node, u32 feature, const char *compatible) 1226173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 1227a1c03319018061304be28d131073ac13a5cb86fbIra Snyder struct fsldma_chan *chan; 12284ce0e953f6286777452bf07c83056342d6b9b257Ira Snyder struct resource res; 1229173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei int err; 1230173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1231173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei /* alloc channel */ 1232a1c03319018061304be28d131073ac13a5cb86fbIra Snyder chan = kzalloc(sizeof(*chan), GFP_KERNEL); 1233a1c03319018061304be28d131073ac13a5cb86fbIra Snyder if (!chan) { 1234e7a29151de1bd52081f27f149b68074fac0323beIra Snyder dev_err(fdev->dev, "no free memory for DMA channels!\n"); 1235e7a29151de1bd52081f27f149b68074fac0323beIra Snyder err = -ENOMEM; 1236e7a29151de1bd52081f27f149b68074fac0323beIra Snyder goto out_return; 1237e7a29151de1bd52081f27f149b68074fac0323beIra Snyder } 1238e7a29151de1bd52081f27f149b68074fac0323beIra Snyder 1239e7a29151de1bd52081f27f149b68074fac0323beIra Snyder /* ioremap registers for use */ 1240a1c03319018061304be28d131073ac13a5cb86fbIra Snyder chan->regs = of_iomap(node, 0); 1241a1c03319018061304be28d131073ac13a5cb86fbIra Snyder if (!chan->regs) { 1242e7a29151de1bd52081f27f149b68074fac0323beIra Snyder dev_err(fdev->dev, "unable to ioremap registers\n"); 1243e7a29151de1bd52081f27f149b68074fac0323beIra Snyder err = -ENOMEM; 1244a1c03319018061304be28d131073ac13a5cb86fbIra Snyder goto out_free_chan; 1245173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei } 1246173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 12474ce0e953f6286777452bf07c83056342d6b9b257Ira Snyder err = of_address_to_resource(node, 0, &res); 1248173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei if (err) { 1249e7a29151de1bd52081f27f149b68074fac0323beIra Snyder dev_err(fdev->dev, "unable to find 'reg' property\n"); 1250e7a29151de1bd52081f27f149b68074fac0323beIra Snyder goto out_iounmap_regs; 1251173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei } 1252173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1253a1c03319018061304be28d131073ac13a5cb86fbIra Snyder chan->feature = feature; 1254173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei if (!fdev->feature) 1255a1c03319018061304be28d131073ac13a5cb86fbIra Snyder fdev->feature = chan->feature; 1256173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1257e7a29151de1bd52081f27f149b68074fac0323beIra Snyder /* 1258e7a29151de1bd52081f27f149b68074fac0323beIra Snyder * If the DMA device's feature is different than the feature 1259e7a29151de1bd52081f27f149b68074fac0323beIra Snyder * of its channels, report the bug 1260173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei */ 1261a1c03319018061304be28d131073ac13a5cb86fbIra Snyder WARN_ON(fdev->feature != chan->feature); 1262e7a29151de1bd52081f27f149b68074fac0323beIra Snyder 1263a1c03319018061304be28d131073ac13a5cb86fbIra Snyder chan->dev = fdev->dev; 1264a1c03319018061304be28d131073ac13a5cb86fbIra Snyder chan->id = ((res.start - 0x100) & 0xfff) >> 7; 1265a1c03319018061304be28d131073ac13a5cb86fbIra Snyder if (chan->id >= FSL_DMA_MAX_CHANS_PER_DEVICE) { 1266e7a29151de1bd52081f27f149b68074fac0323beIra Snyder dev_err(fdev->dev, "too many channels for device\n"); 1267173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei err = -EINVAL; 1268e7a29151de1bd52081f27f149b68074fac0323beIra Snyder goto out_iounmap_regs; 1269173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei } 1270173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1271a1c03319018061304be28d131073ac13a5cb86fbIra Snyder fdev->chan[chan->id] = chan; 1272a1c03319018061304be28d131073ac13a5cb86fbIra Snyder tasklet_init(&chan->tasklet, dma_do_tasklet, (unsigned long)chan); 1273b158471ef63bf399165db96e945a828096502d9dIra Snyder snprintf(chan->name, sizeof(chan->name), "chan%d", chan->id); 1274e7a29151de1bd52081f27f149b68074fac0323beIra Snyder 1275e7a29151de1bd52081f27f149b68074fac0323beIra Snyder /* Initialize the channel */ 1276a1c03319018061304be28d131073ac13a5cb86fbIra Snyder dma_init(chan); 1277173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1278173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei /* Clear cdar registers */ 1279a1c03319018061304be28d131073ac13a5cb86fbIra Snyder set_cdar(chan, 0); 1280173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1281a1c03319018061304be28d131073ac13a5cb86fbIra Snyder switch (chan->feature & FSL_DMA_IP_MASK) { 1282173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei case FSL_DMA_IP_85XX: 1283a1c03319018061304be28d131073ac13a5cb86fbIra Snyder chan->toggle_ext_pause = fsl_chan_toggle_ext_pause; 1284173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei case FSL_DMA_IP_83XX: 1285a1c03319018061304be28d131073ac13a5cb86fbIra Snyder chan->toggle_ext_start = fsl_chan_toggle_ext_start; 1286a1c03319018061304be28d131073ac13a5cb86fbIra Snyder chan->set_src_loop_size = fsl_chan_set_src_loop_size; 1287a1c03319018061304be28d131073ac13a5cb86fbIra Snyder chan->set_dst_loop_size = fsl_chan_set_dst_loop_size; 1288a1c03319018061304be28d131073ac13a5cb86fbIra Snyder chan->set_request_count = fsl_chan_set_request_count; 1289173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei } 1290173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1291a1c03319018061304be28d131073ac13a5cb86fbIra Snyder spin_lock_init(&chan->desc_lock); 12929c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder INIT_LIST_HEAD(&chan->ld_pending); 12939c3a50b7d7ec45da34e73cac66cde12dd6092dd8Ira Snyder INIT_LIST_HEAD(&chan->ld_running); 1294f04cd40701deace2efb9edd7120e59366bda2118Ira Snyder chan->idle = true; 1295173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1296a1c03319018061304be28d131073ac13a5cb86fbIra Snyder chan->common.device = &fdev->common; 12978ac695463f37af902e953d575d3f782e32e170daRussell King - ARM Linux dma_cookie_init(&chan->common); 1298173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1299d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder /* find the IRQ line, if it exists in the device tree */ 1300a1c03319018061304be28d131073ac13a5cb86fbIra Snyder chan->irq = irq_of_parse_and_map(node, 0); 1301d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder 1302173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei /* Add the channel to DMA device channel list */ 1303a1c03319018061304be28d131073ac13a5cb86fbIra Snyder list_add_tail(&chan->common.device_node, &fdev->common.channels); 1304173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei fdev->common.chancnt++; 1305173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1306a1c03319018061304be28d131073ac13a5cb86fbIra Snyder dev_info(fdev->dev, "#%d (%s), irq %d\n", chan->id, compatible, 1307a1c03319018061304be28d131073ac13a5cb86fbIra Snyder chan->irq != NO_IRQ ? chan->irq : fdev->irq); 1308173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1309173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei return 0; 131051ee87f27a1d2c0e08492924f2fb0223c4c704d9Li Yang 1311e7a29151de1bd52081f27f149b68074fac0323beIra Snyderout_iounmap_regs: 1312a1c03319018061304be28d131073ac13a5cb86fbIra Snyder iounmap(chan->regs); 1313a1c03319018061304be28d131073ac13a5cb86fbIra Snyderout_free_chan: 1314a1c03319018061304be28d131073ac13a5cb86fbIra Snyder kfree(chan); 1315e7a29151de1bd52081f27f149b68074fac0323beIra Snyderout_return: 1316173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei return err; 1317173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 1318173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1319a1c03319018061304be28d131073ac13a5cb86fbIra Snyderstatic void fsl_dma_chan_remove(struct fsldma_chan *chan) 1320173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 1321a1c03319018061304be28d131073ac13a5cb86fbIra Snyder irq_dispose_mapping(chan->irq); 1322a1c03319018061304be28d131073ac13a5cb86fbIra Snyder list_del(&chan->common.device_node); 1323a1c03319018061304be28d131073ac13a5cb86fbIra Snyder iounmap(chan->regs); 1324a1c03319018061304be28d131073ac13a5cb86fbIra Snyder kfree(chan); 1325173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 1326173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1327000061245a6797d542854106463b6b20fbdcb12eGrant Likelystatic int __devinit fsldma_of_probe(struct platform_device *op) 1328173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 1329a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder struct fsldma_device *fdev; 133077cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi struct device_node *child; 1331e7a29151de1bd52081f27f149b68074fac0323beIra Snyder int err; 1332173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1333a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder fdev = kzalloc(sizeof(*fdev), GFP_KERNEL); 1334173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei if (!fdev) { 1335e7a29151de1bd52081f27f149b68074fac0323beIra Snyder dev_err(&op->dev, "No enough memory for 'priv'\n"); 1336e7a29151de1bd52081f27f149b68074fac0323beIra Snyder err = -ENOMEM; 1337e7a29151de1bd52081f27f149b68074fac0323beIra Snyder goto out_return; 1338173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei } 1339e7a29151de1bd52081f27f149b68074fac0323beIra Snyder 1340e7a29151de1bd52081f27f149b68074fac0323beIra Snyder fdev->dev = &op->dev; 1341173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei INIT_LIST_HEAD(&fdev->common.channels); 1342173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1343e7a29151de1bd52081f27f149b68074fac0323beIra Snyder /* ioremap the registers for use */ 134461c7a080a5a061c976988fd4b844dfb468dda255Grant Likely fdev->regs = of_iomap(op->dev.of_node, 0); 1345e7a29151de1bd52081f27f149b68074fac0323beIra Snyder if (!fdev->regs) { 1346e7a29151de1bd52081f27f149b68074fac0323beIra Snyder dev_err(&op->dev, "unable to ioremap registers\n"); 1347e7a29151de1bd52081f27f149b68074fac0323beIra Snyder err = -ENOMEM; 1348e7a29151de1bd52081f27f149b68074fac0323beIra Snyder goto out_free_fdev; 1349173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei } 1350173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1351d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder /* map the channel IRQ if it exists, but don't hookup the handler yet */ 135261c7a080a5a061c976988fd4b844dfb468dda255Grant Likely fdev->irq = irq_of_parse_and_map(op->dev.of_node, 0); 1353d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder 1354173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask); 1355173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei dma_cap_set(DMA_INTERRUPT, fdev->common.cap_mask); 1356c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder dma_cap_set(DMA_SG, fdev->common.cap_mask); 1357bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder dma_cap_set(DMA_SLAVE, fdev->common.cap_mask); 1358173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei fdev->common.device_alloc_chan_resources = fsl_dma_alloc_chan_resources; 1359173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei fdev->common.device_free_chan_resources = fsl_dma_free_chan_resources; 13602187c269ad29510f1d65ec684133d1d3426d0eedZhang Wei fdev->common.device_prep_dma_interrupt = fsl_dma_prep_interrupt; 1361173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei fdev->common.device_prep_dma_memcpy = fsl_dma_prep_memcpy; 1362c14330417ef2050f4bf38ac20e125785fea14351Ira Snyder fdev->common.device_prep_dma_sg = fsl_dma_prep_sg; 13630793448187643b50af89d36b08470baf45a3cab4Linus Walleij fdev->common.device_tx_status = fsl_tx_status; 1364173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei fdev->common.device_issue_pending = fsl_dma_memcpy_issue_pending; 1365bbea0b6e0d214ef1511b9c6ccf3af26b38f0af7dIra Snyder fdev->common.device_prep_slave_sg = fsl_dma_prep_slave_sg; 1366c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij fdev->common.device_control = fsl_dma_device_control; 1367e7a29151de1bd52081f27f149b68074fac0323beIra Snyder fdev->common.dev = &op->dev; 1368173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1369e2c8e425baa01a4c8e6ae1b90194ed3d3cde0c66Li Yang dma_set_mask(&(op->dev), DMA_BIT_MASK(36)); 1370e2c8e425baa01a4c8e6ae1b90194ed3d3cde0c66Li Yang 1371e7a29151de1bd52081f27f149b68074fac0323beIra Snyder dev_set_drvdata(&op->dev, fdev); 137277cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi 1373e7a29151de1bd52081f27f149b68074fac0323beIra Snyder /* 1374e7a29151de1bd52081f27f149b68074fac0323beIra Snyder * We cannot use of_platform_bus_probe() because there is no 1375e7a29151de1bd52081f27f149b68074fac0323beIra Snyder * of_platform_bus_remove(). Instead, we manually instantiate every DMA 137677cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi * channel object. 137777cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi */ 137861c7a080a5a061c976988fd4b844dfb468dda255Grant Likely for_each_child_of_node(op->dev.of_node, child) { 1379e7a29151de1bd52081f27f149b68074fac0323beIra Snyder if (of_device_is_compatible(child, "fsl,eloplus-dma-channel")) { 138077cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi fsl_dma_chan_probe(fdev, child, 138177cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi FSL_DMA_IP_85XX | FSL_DMA_BIG_ENDIAN, 138277cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi "fsl,eloplus-dma-channel"); 1383e7a29151de1bd52081f27f149b68074fac0323beIra Snyder } 1384e7a29151de1bd52081f27f149b68074fac0323beIra Snyder 1385e7a29151de1bd52081f27f149b68074fac0323beIra Snyder if (of_device_is_compatible(child, "fsl,elo-dma-channel")) { 138677cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi fsl_dma_chan_probe(fdev, child, 138777cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi FSL_DMA_IP_83XX | FSL_DMA_LITTLE_ENDIAN, 138877cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi "fsl,elo-dma-channel"); 1389e7a29151de1bd52081f27f149b68074fac0323beIra Snyder } 139077cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi } 1391173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1392d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder /* 1393d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder * Hookup the IRQ handler(s) 1394d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder * 1395d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder * If we have a per-controller interrupt, we prefer that to the 1396d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder * per-channel interrupts to reduce the number of shared interrupt 1397d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder * handlers on the same IRQ line 1398d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder */ 1399d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder err = fsldma_request_irqs(fdev); 1400d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder if (err) { 1401d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder dev_err(fdev->dev, "unable to request IRQs\n"); 1402d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder goto out_free_fdev; 1403d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder } 1404d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder 1405173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei dma_async_device_register(&fdev->common); 1406173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei return 0; 1407173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1408e7a29151de1bd52081f27f149b68074fac0323beIra Snyderout_free_fdev: 1409d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder irq_dispose_mapping(fdev->irq); 1410173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei kfree(fdev); 1411e7a29151de1bd52081f27f149b68074fac0323beIra Snyderout_return: 1412173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei return err; 1413173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 1414173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 14152dc11581376829303b98eadb2de253bee065a56aGrant Likelystatic int fsldma_of_remove(struct platform_device *op) 141677cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi{ 1417a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder struct fsldma_device *fdev; 141877cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi unsigned int i; 141977cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi 1420e7a29151de1bd52081f27f149b68074fac0323beIra Snyder fdev = dev_get_drvdata(&op->dev); 142177cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi dma_async_device_unregister(&fdev->common); 142277cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi 1423d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder fsldma_free_irqs(fdev); 1424d3f620b2c4fecdc8e060b70e8d92d29fc01c6126Ira Snyder 1425e7a29151de1bd52081f27f149b68074fac0323beIra Snyder for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { 142677cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi if (fdev->chan[i]) 142777cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi fsl_dma_chan_remove(fdev->chan[i]); 1428e7a29151de1bd52081f27f149b68074fac0323beIra Snyder } 142977cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi 1430e7a29151de1bd52081f27f149b68074fac0323beIra Snyder iounmap(fdev->regs); 1431e7a29151de1bd52081f27f149b68074fac0323beIra Snyder dev_set_drvdata(&op->dev, NULL); 143277cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi kfree(fdev); 143377cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi 143477cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi return 0; 143577cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi} 143677cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi 14374b1cf1facca31b7db2a61d8aa2ba40d5a93a0957Márton Némethstatic const struct of_device_id fsldma_of_ids[] = { 1438049c9d45531d9825bf737891163a794fca1421c5Kumar Gala { .compatible = "fsl,eloplus-dma", }, 1439049c9d45531d9825bf737891163a794fca1421c5Kumar Gala { .compatible = "fsl,elo-dma", }, 1440173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei {} 1441173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}; 1442173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 14438faa7cf828bca1745a4ed599876567f5afc47544Ira W. Snyderstatic struct platform_driver fsldma_of_driver = { 14444018294b53d1dae026880e45f174c1cc63b5d435Grant Likely .driver = { 14454018294b53d1dae026880e45f174c1cc63b5d435Grant Likely .name = "fsl-elo-dma", 14464018294b53d1dae026880e45f174c1cc63b5d435Grant Likely .owner = THIS_MODULE, 14474018294b53d1dae026880e45f174c1cc63b5d435Grant Likely .of_match_table = fsldma_of_ids, 14484018294b53d1dae026880e45f174c1cc63b5d435Grant Likely }, 14494018294b53d1dae026880e45f174c1cc63b5d435Grant Likely .probe = fsldma_of_probe, 14504018294b53d1dae026880e45f174c1cc63b5d435Grant Likely .remove = fsldma_of_remove, 1451173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei}; 1452173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1453a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder/*----------------------------------------------------------------------------*/ 1454a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder/* Module Init / Exit */ 1455a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder/*----------------------------------------------------------------------------*/ 1456a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyder 1457a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyderstatic __init int fsldma_init(void) 1458173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei{ 145977cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi pr_info("Freescale Elo / Elo Plus DMA driver\n"); 1460000061245a6797d542854106463b6b20fbdcb12eGrant Likely return platform_driver_register(&fsldma_of_driver); 146177cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi} 146277cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi 1463a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snyderstatic void __exit fsldma_exit(void) 146477cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi{ 1465000061245a6797d542854106463b6b20fbdcb12eGrant Likely platform_driver_unregister(&fsldma_of_driver); 1466173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei} 1467173acc7ce8538f1f3040791dc622a92aadc12cf4Zhang Wei 1468a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snydersubsys_initcall(fsldma_init); 1469a4f56d4b103d4e5d1a59a9118db0185a6bd1a83bIra Snydermodule_exit(fsldma_exit); 147077cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur Tabi 147177cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur TabiMODULE_DESCRIPTION("Freescale Elo / Elo Plus DMA driver"); 147277cd62e8082b9743b59ee1946a4c3ee2e3cd2bceTimur TabiMODULE_LICENSE("GPL"); 1473