1ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto/* 2ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * Driver for the TXx9 SoC DMA Controller 3ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * 4ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * Copyright (C) 2009 Atsushi Nemoto 5ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * 6ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * This program is free software; you can redistribute it and/or modify 7ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * it under the terms of the GNU General Public License version 2 as 8ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * published by the Free Software Foundation. 9ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto */ 10ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#include <linux/dma-mapping.h> 11ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#include <linux/init.h> 12ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#include <linux/interrupt.h> 13ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#include <linux/io.h> 14ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#include <linux/module.h> 15ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#include <linux/platform_device.h> 16ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#include <linux/slab.h> 17ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#include <linux/scatterlist.h> 18d2ebfb335b0426deb1a4fb14e4e926d81ecd8235Russell King - ARM Linux 19d2ebfb335b0426deb1a4fb14e4e926d81ecd8235Russell King - ARM Linux#include "dmaengine.h" 20ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#include "txx9dmac.h" 21ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 22ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic struct txx9dmac_chan *to_txx9dmac_chan(struct dma_chan *chan) 23ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 24ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return container_of(chan, struct txx9dmac_chan, chan); 25ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 26ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 27ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic struct txx9dmac_cregs __iomem *__dma_regs(const struct txx9dmac_chan *dc) 28ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 29ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return dc->ch_regs; 30ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 31ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 32ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic struct txx9dmac_cregs32 __iomem *__dma_regs32( 33ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto const struct txx9dmac_chan *dc) 34ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 35ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return dc->ch_regs; 36ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 37ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 38ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#define channel64_readq(dc, name) \ 39ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto __raw_readq(&(__dma_regs(dc)->name)) 40ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#define channel64_writeq(dc, name, val) \ 41ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto __raw_writeq((val), &(__dma_regs(dc)->name)) 42ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#define channel64_readl(dc, name) \ 43ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto __raw_readl(&(__dma_regs(dc)->name)) 44ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#define channel64_writel(dc, name, val) \ 45ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto __raw_writel((val), &(__dma_regs(dc)->name)) 46ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 47ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#define channel32_readl(dc, name) \ 48ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto __raw_readl(&(__dma_regs32(dc)->name)) 49ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#define channel32_writel(dc, name, val) \ 50ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto __raw_writel((val), &(__dma_regs32(dc)->name)) 51ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 52ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#define channel_readq(dc, name) channel64_readq(dc, name) 53ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#define channel_writeq(dc, name, val) channel64_writeq(dc, name, val) 54ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#define channel_readl(dc, name) \ 55ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto (is_dmac64(dc) ? \ 56ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_readl(dc, name) : channel32_readl(dc, name)) 57ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#define channel_writel(dc, name, val) \ 58ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto (is_dmac64(dc) ? \ 59ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_writel(dc, name, val) : channel32_writel(dc, name, val)) 60ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 61ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic dma_addr_t channel64_read_CHAR(const struct txx9dmac_chan *dc) 62ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 63ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (sizeof(__dma_regs(dc)->CHAR) == sizeof(u64)) 64ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return channel64_readq(dc, CHAR); 65ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto else 66ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return channel64_readl(dc, CHAR); 67ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 68ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 69ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void channel64_write_CHAR(const struct txx9dmac_chan *dc, dma_addr_t val) 70ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 71ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (sizeof(__dma_regs(dc)->CHAR) == sizeof(u64)) 72ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_writeq(dc, CHAR, val); 73ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto else 74ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_writel(dc, CHAR, val); 75ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 76ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 77ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void channel64_clear_CHAR(const struct txx9dmac_chan *dc) 78ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 79ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#if defined(CONFIG_32BIT) && !defined(CONFIG_64BIT_PHYS_ADDR) 80ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_writel(dc, CHAR, 0); 81ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_writel(dc, __pad_CHAR, 0); 82ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#else 83ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_writeq(dc, CHAR, 0); 84ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#endif 85ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 86ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 87ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic dma_addr_t channel_read_CHAR(const struct txx9dmac_chan *dc) 88ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 89ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (is_dmac64(dc)) 90ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return channel64_read_CHAR(dc); 91ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto else 92ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return channel32_readl(dc, CHAR); 93ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 94ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 95ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void channel_write_CHAR(const struct txx9dmac_chan *dc, dma_addr_t val) 96ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 97ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (is_dmac64(dc)) 98ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_write_CHAR(dc, val); 99ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto else 100ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel32_writel(dc, CHAR, val); 101ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 102ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 103ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic struct txx9dmac_regs __iomem *__txx9dmac_regs( 104ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto const struct txx9dmac_dev *ddev) 105ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 106ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return ddev->regs; 107ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 108ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 109ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic struct txx9dmac_regs32 __iomem *__txx9dmac_regs32( 110ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto const struct txx9dmac_dev *ddev) 111ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 112ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return ddev->regs; 113ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 114ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 115ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#define dma64_readl(ddev, name) \ 116ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto __raw_readl(&(__txx9dmac_regs(ddev)->name)) 117ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#define dma64_writel(ddev, name, val) \ 118ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto __raw_writel((val), &(__txx9dmac_regs(ddev)->name)) 119ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 120ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#define dma32_readl(ddev, name) \ 121ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto __raw_readl(&(__txx9dmac_regs32(ddev)->name)) 122ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#define dma32_writel(ddev, name, val) \ 123ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto __raw_writel((val), &(__txx9dmac_regs32(ddev)->name)) 124ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 125ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#define dma_readl(ddev, name) \ 126ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto (__is_dmac64(ddev) ? \ 127ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma64_readl(ddev, name) : dma32_readl(ddev, name)) 128ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#define dma_writel(ddev, name, val) \ 129ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto (__is_dmac64(ddev) ? \ 130ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma64_writel(ddev, name, val) : dma32_writel(ddev, name, val)) 131ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 132ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic struct device *chan2dev(struct dma_chan *chan) 133ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 134ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return &chan->dev->device; 135ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 136ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic struct device *chan2parent(struct dma_chan *chan) 137ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 138ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return chan->dev->device.parent; 139ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 140ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 141ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic struct txx9dmac_desc * 142ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemototxd_to_txx9dmac_desc(struct dma_async_tx_descriptor *txd) 143ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 144ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return container_of(txd, struct txx9dmac_desc, txd); 145ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 146ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 147ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic dma_addr_t desc_read_CHAR(const struct txx9dmac_chan *dc, 148ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto const struct txx9dmac_desc *desc) 149ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 150ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return is_dmac64(dc) ? desc->hwdesc.CHAR : desc->hwdesc32.CHAR; 151ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 152ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 153ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void desc_write_CHAR(const struct txx9dmac_chan *dc, 154ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *desc, dma_addr_t val) 155ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 156ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (is_dmac64(dc)) 157ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->hwdesc.CHAR = val; 158ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto else 159ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->hwdesc32.CHAR = val; 160ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 161ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 162ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#define TXX9_DMA_MAX_COUNT 0x04000000 163ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 164ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#define TXX9_DMA_INITIAL_DESC_COUNT 64 165ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 166ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic struct txx9dmac_desc *txx9dmac_first_active(struct txx9dmac_chan *dc) 167ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 168ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return list_entry(dc->active_list.next, 169ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc, desc_node); 170ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 171ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 172ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic struct txx9dmac_desc *txx9dmac_last_active(struct txx9dmac_chan *dc) 173ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 174ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return list_entry(dc->active_list.prev, 175ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc, desc_node); 176ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 177ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 178ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic struct txx9dmac_desc *txx9dmac_first_queued(struct txx9dmac_chan *dc) 179ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 180ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return list_entry(dc->queue.next, struct txx9dmac_desc, desc_node); 181ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 182ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 183ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic struct txx9dmac_desc *txx9dmac_last_child(struct txx9dmac_desc *desc) 184ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 1851979b186b80449ac6574d97c254b694c8a99b703Dan Williams if (!list_empty(&desc->tx_list)) 1861979b186b80449ac6574d97c254b694c8a99b703Dan Williams desc = list_entry(desc->tx_list.prev, typeof(*desc), desc_node); 187ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return desc; 188ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 189ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 190ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic dma_cookie_t txx9dmac_tx_submit(struct dma_async_tx_descriptor *tx); 191ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 192ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic struct txx9dmac_desc *txx9dmac_desc_alloc(struct txx9dmac_chan *dc, 193ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto gfp_t flags) 194ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 195ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_dev *ddev = dc->ddev; 196ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *desc; 197ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 198ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc = kzalloc(sizeof(*desc), flags); 199ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!desc) 200ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return NULL; 2011979b186b80449ac6574d97c254b694c8a99b703Dan Williams INIT_LIST_HEAD(&desc->tx_list); 202ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_async_tx_descriptor_init(&desc->txd, &dc->chan); 203ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->txd.tx_submit = txx9dmac_tx_submit; 204ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* txd.flags will be overwritten in prep funcs */ 205ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->txd.flags = DMA_CTRL_ACK; 206ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->txd.phys = dma_map_single(chan2parent(&dc->chan), &desc->hwdesc, 207ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto ddev->descsize, DMA_TO_DEVICE); 208ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return desc; 209ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 210ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 211ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic struct txx9dmac_desc *txx9dmac_desc_get(struct txx9dmac_chan *dc) 212ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 213ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *desc, *_desc; 214ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *ret = NULL; 215ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto unsigned int i = 0; 216ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 217ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_lock_bh(&dc->lock); 218ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto list_for_each_entry_safe(desc, _desc, &dc->free_list, desc_node) { 219ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (async_tx_test_ack(&desc->txd)) { 220ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto list_del(&desc->desc_node); 221ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto ret = desc; 222ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto break; 223ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 224ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_dbg(chan2dev(&dc->chan), "desc %p not ACKed\n", desc); 225ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto i++; 226ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 227ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_unlock_bh(&dc->lock); 228ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 229ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_vdbg(chan2dev(&dc->chan), "scanned %u descriptors on freelist\n", 230ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto i); 231ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!ret) { 232ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto ret = txx9dmac_desc_alloc(dc, GFP_ATOMIC); 233ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (ret) { 234ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_lock_bh(&dc->lock); 235ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->descs_allocated++; 236ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_unlock_bh(&dc->lock); 237ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } else 238ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_err(chan2dev(&dc->chan), 239ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto "not enough descriptors available\n"); 240ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 241ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return ret; 242ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 243ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 244ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void txx9dmac_sync_desc_for_cpu(struct txx9dmac_chan *dc, 245ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *desc) 246ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 247ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_dev *ddev = dc->ddev; 248ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *child; 249ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 2501979b186b80449ac6574d97c254b694c8a99b703Dan Williams list_for_each_entry(child, &desc->tx_list, desc_node) 251ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_sync_single_for_cpu(chan2parent(&dc->chan), 252ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto child->txd.phys, ddev->descsize, 253ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto DMA_TO_DEVICE); 254ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_sync_single_for_cpu(chan2parent(&dc->chan), 255ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->txd.phys, ddev->descsize, 256ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto DMA_TO_DEVICE); 257ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 258ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 259ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto/* 260ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * Move a descriptor, including any children, to the free list. 261ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * `desc' must not be on any lists. 262ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto */ 263ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void txx9dmac_desc_put(struct txx9dmac_chan *dc, 264ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *desc) 265ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 266ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (desc) { 267ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *child; 268ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 269ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_sync_desc_for_cpu(dc, desc); 270ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 271ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_lock_bh(&dc->lock); 2721979b186b80449ac6574d97c254b694c8a99b703Dan Williams list_for_each_entry(child, &desc->tx_list, desc_node) 273ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_vdbg(chan2dev(&dc->chan), 274ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto "moving child desc %p to freelist\n", 275ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto child); 2761979b186b80449ac6574d97c254b694c8a99b703Dan Williams list_splice_init(&desc->tx_list, &dc->free_list); 277ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_vdbg(chan2dev(&dc->chan), "moving desc %p to freelist\n", 278ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc); 279ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto list_add(&desc->desc_node, &dc->free_list); 280ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_unlock_bh(&dc->lock); 281ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 282ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 283ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 284ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto/*----------------------------------------------------------------------*/ 285ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 286ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void txx9dmac_dump_regs(struct txx9dmac_chan *dc) 287ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 288ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (is_dmac64(dc)) 289ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_err(chan2dev(&dc->chan), 290ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto " CHAR: %#llx SAR: %#llx DAR: %#llx CNTR: %#x" 291ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto " SAIR: %#x DAIR: %#x CCR: %#x CSR: %#x\n", 292ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto (u64)channel64_read_CHAR(dc), 293ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_readq(dc, SAR), 294ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_readq(dc, DAR), 295ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_readl(dc, CNTR), 296ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_readl(dc, SAIR), 297ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_readl(dc, DAIR), 298ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_readl(dc, CCR), 299ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_readl(dc, CSR)); 300ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto else 301ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_err(chan2dev(&dc->chan), 302ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto " CHAR: %#x SAR: %#x DAR: %#x CNTR: %#x" 303ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto " SAIR: %#x DAIR: %#x CCR: %#x CSR: %#x\n", 304ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel32_readl(dc, CHAR), 305ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel32_readl(dc, SAR), 306ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel32_readl(dc, DAR), 307ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel32_readl(dc, CNTR), 308ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel32_readl(dc, SAIR), 309ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel32_readl(dc, DAIR), 310ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel32_readl(dc, CCR), 311ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel32_readl(dc, CSR)); 312ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 313ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 314ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void txx9dmac_reset_chan(struct txx9dmac_chan *dc) 315ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 316ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel_writel(dc, CCR, TXX9_DMA_CCR_CHRST); 317ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (is_dmac64(dc)) { 318ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_clear_CHAR(dc); 319ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel_writeq(dc, SAR, 0); 320ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel_writeq(dc, DAR, 0); 321ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } else { 322ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel_writel(dc, CHAR, 0); 323ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel_writel(dc, SAR, 0); 324ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel_writel(dc, DAR, 0); 325ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 326ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel_writel(dc, CNTR, 0); 327ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel_writel(dc, SAIR, 0); 328ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel_writel(dc, DAIR, 0); 329ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel_writel(dc, CCR, 0); 330ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto mmiowb(); 331ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 332ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 333ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto/* Called with dc->lock held and bh disabled */ 334ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void txx9dmac_dostart(struct txx9dmac_chan *dc, 335ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *first) 336ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 337ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_slave *ds = dc->chan.private; 338ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto u32 sai, dai; 339ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 340ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_vdbg(chan2dev(&dc->chan), "dostart %u %p\n", 341ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto first->txd.cookie, first); 342ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* ASSERT: channel is idle */ 343ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (channel_readl(dc, CSR) & TXX9_DMA_CSR_XFACT) { 344ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_err(chan2dev(&dc->chan), 345ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto "BUG: Attempted to start non-idle channel\n"); 346ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_dump_regs(dc); 347ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* The tasklet will hopefully advance the queue... */ 348ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return; 349ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 350ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 351ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (is_dmac64(dc)) { 352ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_writel(dc, CNTR, 0); 353ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_writel(dc, CSR, 0xffffffff); 354ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (ds) { 355ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (ds->tx_reg) { 356ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto sai = ds->reg_width; 357ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dai = 0; 358ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } else { 359ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto sai = 0; 360ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dai = ds->reg_width; 361ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 362ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } else { 363ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto sai = 8; 364ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dai = 8; 365ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 366ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_writel(dc, SAIR, sai); 367ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_writel(dc, DAIR, dai); 368ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* All 64-bit DMAC supports SMPCHN */ 369ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_writel(dc, CCR, dc->ccr); 370ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* Writing a non zero value to CHAR will assert XFACT */ 371ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_write_CHAR(dc, first->txd.phys); 372ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } else { 373ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel32_writel(dc, CNTR, 0); 374ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel32_writel(dc, CSR, 0xffffffff); 375ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (ds) { 376ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (ds->tx_reg) { 377ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto sai = ds->reg_width; 378ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dai = 0; 379ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } else { 380ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto sai = 0; 381ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dai = ds->reg_width; 382ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 383ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } else { 384ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto sai = 4; 385ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dai = 4; 386ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 387ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel32_writel(dc, SAIR, sai); 388ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel32_writel(dc, DAIR, dai); 389ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (txx9_dma_have_SMPCHN()) { 390ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel32_writel(dc, CCR, dc->ccr); 391ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* Writing a non zero value to CHAR will assert XFACT */ 392ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel32_writel(dc, CHAR, first->txd.phys); 393ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } else { 394ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel32_writel(dc, CHAR, first->txd.phys); 395ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel32_writel(dc, CCR, dc->ccr); 396ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 397ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 398ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 399ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 400ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto/*----------------------------------------------------------------------*/ 401ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 402ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void 403ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemototxx9dmac_descriptor_complete(struct txx9dmac_chan *dc, 404ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *desc) 405ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 406ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_async_tx_callback callback; 407ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto void *param; 408ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct dma_async_tx_descriptor *txd = &desc->txd; 409ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_slave *ds = dc->chan.private; 410ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 411ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_vdbg(chan2dev(&dc->chan), "descriptor %u %p complete\n", 412ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txd->cookie, desc); 413ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 414f7fbce07c6ce26a25b4e0cb5f241c361fde87901Russell King - ARM Linux dma_cookie_complete(txd); 415ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto callback = txd->callback; 416ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto param = txd->callback_param; 417ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 418ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_sync_desc_for_cpu(dc, desc); 4191979b186b80449ac6574d97c254b694c8a99b703Dan Williams list_splice_init(&desc->tx_list, &dc->free_list); 420ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto list_move(&desc->desc_node, &dc->free_list); 421ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 422ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!ds) { 423ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_addr_t dmaaddr; 424ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) { 425ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dmaaddr = is_dmac64(dc) ? 426ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->hwdesc.DAR : desc->hwdesc32.DAR; 4274ac4aa5cc3b00cc558575065ae71043e92d1a69aAtsushi Nemoto if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE) 4284ac4aa5cc3b00cc558575065ae71043e92d1a69aAtsushi Nemoto dma_unmap_single(chan2parent(&dc->chan), 4294ac4aa5cc3b00cc558575065ae71043e92d1a69aAtsushi Nemoto dmaaddr, desc->len, DMA_FROM_DEVICE); 4304ac4aa5cc3b00cc558575065ae71043e92d1a69aAtsushi Nemoto else 4314ac4aa5cc3b00cc558575065ae71043e92d1a69aAtsushi Nemoto dma_unmap_page(chan2parent(&dc->chan), 4324ac4aa5cc3b00cc558575065ae71043e92d1a69aAtsushi Nemoto dmaaddr, desc->len, DMA_FROM_DEVICE); 433ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 434ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) { 435ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dmaaddr = is_dmac64(dc) ? 436ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->hwdesc.SAR : desc->hwdesc32.SAR; 4374ac4aa5cc3b00cc558575065ae71043e92d1a69aAtsushi Nemoto if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE) 4384ac4aa5cc3b00cc558575065ae71043e92d1a69aAtsushi Nemoto dma_unmap_single(chan2parent(&dc->chan), 4394ac4aa5cc3b00cc558575065ae71043e92d1a69aAtsushi Nemoto dmaaddr, desc->len, DMA_TO_DEVICE); 4404ac4aa5cc3b00cc558575065ae71043e92d1a69aAtsushi Nemoto else 4414ac4aa5cc3b00cc558575065ae71043e92d1a69aAtsushi Nemoto dma_unmap_page(chan2parent(&dc->chan), 4424ac4aa5cc3b00cc558575065ae71043e92d1a69aAtsushi Nemoto dmaaddr, desc->len, DMA_TO_DEVICE); 443ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 444ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 445ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 446ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* 447ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * The API requires that no submissions are done from a 448ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * callback, so we don't need to drop the lock here 449ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto */ 450ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (callback) 451ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto callback(param); 452ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_run_dependencies(txd); 453ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 454ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 455ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void txx9dmac_dequeue(struct txx9dmac_chan *dc, struct list_head *list) 456ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 457ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_dev *ddev = dc->ddev; 458ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *desc; 459ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *prev = NULL; 460ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 461ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto BUG_ON(!list_empty(list)); 462ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto do { 463ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc = txx9dmac_first_queued(dc); 464ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (prev) { 465ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc_write_CHAR(dc, prev, desc->txd.phys); 466ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_sync_single_for_device(chan2parent(&dc->chan), 467ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto prev->txd.phys, ddev->descsize, 468ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto DMA_TO_DEVICE); 469ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 470ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto prev = txx9dmac_last_child(desc); 471ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto list_move_tail(&desc->desc_node, list); 472ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* Make chain-completion interrupt happen */ 473ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if ((desc->txd.flags & DMA_PREP_INTERRUPT) && 474ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto !txx9dmac_chan_INTENT(dc)) 475ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto break; 476ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } while (!list_empty(&dc->queue)); 477ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 478ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 479ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void txx9dmac_complete_all(struct txx9dmac_chan *dc) 480ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 481ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *desc, *_desc; 482ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto LIST_HEAD(list); 483ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 484ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* 485ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * Submit queued descriptors ASAP, i.e. before we go through 486ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * the completed ones. 487ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto */ 488ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto list_splice_init(&dc->active_list, &list); 489ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!list_empty(&dc->queue)) { 490ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_dequeue(dc, &dc->active_list); 491ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_dostart(dc, txx9dmac_first_active(dc)); 492ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 493ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 494ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto list_for_each_entry_safe(desc, _desc, &list, desc_node) 495ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_descriptor_complete(dc, desc); 496ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 497ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 498ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void txx9dmac_dump_desc(struct txx9dmac_chan *dc, 499ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_hwdesc *desc) 500ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 501ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (is_dmac64(dc)) { 502ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#ifdef TXX9_DMA_USE_SIMPLE_CHAIN 503ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_crit(chan2dev(&dc->chan), 504ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto " desc: ch%#llx s%#llx d%#llx c%#x\n", 505ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto (u64)desc->CHAR, desc->SAR, desc->DAR, desc->CNTR); 506ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#else 507ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_crit(chan2dev(&dc->chan), 508ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto " desc: ch%#llx s%#llx d%#llx c%#x" 509ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto " si%#x di%#x cc%#x cs%#x\n", 510ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto (u64)desc->CHAR, desc->SAR, desc->DAR, desc->CNTR, 511ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->SAIR, desc->DAIR, desc->CCR, desc->CSR); 512ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#endif 513ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } else { 514ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_hwdesc32 *d = (struct txx9dmac_hwdesc32 *)desc; 515ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#ifdef TXX9_DMA_USE_SIMPLE_CHAIN 516ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_crit(chan2dev(&dc->chan), 517ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto " desc: ch%#x s%#x d%#x c%#x\n", 518ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto d->CHAR, d->SAR, d->DAR, d->CNTR); 519ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#else 520ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_crit(chan2dev(&dc->chan), 521ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto " desc: ch%#x s%#x d%#x c%#x" 522ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto " si%#x di%#x cc%#x cs%#x\n", 523ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto d->CHAR, d->SAR, d->DAR, d->CNTR, 524ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto d->SAIR, d->DAIR, d->CCR, d->CSR); 525ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto#endif 526ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 527ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 528ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 529ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void txx9dmac_handle_error(struct txx9dmac_chan *dc, u32 csr) 530ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 531ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *bad_desc; 532ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *child; 533ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto u32 errors; 534ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 535ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* 536ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * The descriptor currently at the head of the active list is 537ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * borked. Since we don't have any way to report errors, we'll 538ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * just have to scream loudly and try to carry on. 539ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto */ 540ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_crit(chan2dev(&dc->chan), "Abnormal Chain Completion\n"); 541ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_dump_regs(dc); 542ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 543ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto bad_desc = txx9dmac_first_active(dc); 544ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto list_del_init(&bad_desc->desc_node); 545ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 546ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* Clear all error flags and try to restart the controller */ 547ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto errors = csr & (TXX9_DMA_CSR_ABCHC | 548ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto TXX9_DMA_CSR_CFERR | TXX9_DMA_CSR_CHERR | 549ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto TXX9_DMA_CSR_DESERR | TXX9_DMA_CSR_SORERR); 550ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel_writel(dc, CSR, errors); 551ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 552ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (list_empty(&dc->active_list) && !list_empty(&dc->queue)) 553ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_dequeue(dc, &dc->active_list); 554ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!list_empty(&dc->active_list)) 555ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_dostart(dc, txx9dmac_first_active(dc)); 556ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 557ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_crit(chan2dev(&dc->chan), 558ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto "Bad descriptor submitted for DMA! (cookie: %d)\n", 559ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto bad_desc->txd.cookie); 560ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_dump_desc(dc, &bad_desc->hwdesc); 5611979b186b80449ac6574d97c254b694c8a99b703Dan Williams list_for_each_entry(child, &bad_desc->tx_list, desc_node) 562ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_dump_desc(dc, &child->hwdesc); 563ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* Pretend the descriptor completed successfully */ 564ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_descriptor_complete(dc, bad_desc); 565ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 566ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 567ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void txx9dmac_scan_descriptors(struct txx9dmac_chan *dc) 568ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 569ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_addr_t chain; 570ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *desc, *_desc; 571ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *child; 572ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto u32 csr; 573ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 574ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (is_dmac64(dc)) { 575ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto chain = channel64_read_CHAR(dc); 576ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto csr = channel64_readl(dc, CSR); 577ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel64_writel(dc, CSR, csr); 578ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } else { 579ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto chain = channel32_readl(dc, CHAR); 580ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto csr = channel32_readl(dc, CSR); 581ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel32_writel(dc, CSR, csr); 582ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 583ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* For dynamic chain, we should look at XFACT instead of NCHNC */ 584ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!(csr & (TXX9_DMA_CSR_XFACT | TXX9_DMA_CSR_ABCHC))) { 585ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* Everything we've submitted is done */ 586ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_complete_all(dc); 587ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return; 588ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 589ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!(csr & TXX9_DMA_CSR_CHNEN)) 590ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto chain = 0; /* last descriptor of this chain */ 591ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 592ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_vdbg(chan2dev(&dc->chan), "scan_descriptors: char=%#llx\n", 593ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto (u64)chain); 594ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 595ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto list_for_each_entry_safe(desc, _desc, &dc->active_list, desc_node) { 596ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (desc_read_CHAR(dc, desc) == chain) { 597ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* This one is currently in progress */ 598ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (csr & TXX9_DMA_CSR_ABCHC) 599ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto goto scan_done; 600ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return; 601ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 602ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 6031979b186b80449ac6574d97c254b694c8a99b703Dan Williams list_for_each_entry(child, &desc->tx_list, desc_node) 604ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (desc_read_CHAR(dc, child) == chain) { 605ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* Currently in progress */ 606ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (csr & TXX9_DMA_CSR_ABCHC) 607ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto goto scan_done; 608ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return; 609ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 610ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 611ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* 612ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * No descriptors so far seem to be in progress, i.e. 613ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * this one must be done. 614ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto */ 615ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_descriptor_complete(dc, desc); 616ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 617ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotoscan_done: 618ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (csr & TXX9_DMA_CSR_ABCHC) { 619ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_handle_error(dc, csr); 620ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return; 621ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 622ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 623ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_err(chan2dev(&dc->chan), 624ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto "BUG: All descriptors done, but channel not idle!\n"); 625ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 626ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* Try to continue after resetting the channel... */ 627ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_reset_chan(dc); 628ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 629ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!list_empty(&dc->queue)) { 630ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_dequeue(dc, &dc->active_list); 631ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_dostart(dc, txx9dmac_first_active(dc)); 632ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 633ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 634ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 635ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void txx9dmac_chan_tasklet(unsigned long data) 636ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 637ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto int irq; 638ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto u32 csr; 639ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_chan *dc; 640ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 641ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc = (struct txx9dmac_chan *)data; 642ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto csr = channel_readl(dc, CSR); 643ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_vdbg(chan2dev(&dc->chan), "tasklet: status=%x\n", csr); 644ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 645ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_lock(&dc->lock); 646ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (csr & (TXX9_DMA_CSR_ABCHC | TXX9_DMA_CSR_NCHNC | 647ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto TXX9_DMA_CSR_NTRNFC)) 648ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_scan_descriptors(dc); 649ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_unlock(&dc->lock); 650ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto irq = dc->irq; 651ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 652ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto enable_irq(irq); 653ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 654ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 655ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic irqreturn_t txx9dmac_chan_interrupt(int irq, void *dev_id) 656ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 657ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_chan *dc = dev_id; 658ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 659ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_vdbg(chan2dev(&dc->chan), "interrupt: status=%#x\n", 660ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel_readl(dc, CSR)); 661ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 662ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto tasklet_schedule(&dc->tasklet); 663ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* 664ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * Just disable the interrupts. We'll turn them back on in the 665ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * softirq handler. 666ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto */ 667ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto disable_irq_nosync(irq); 668ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 669ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return IRQ_HANDLED; 670ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 671ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 672ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void txx9dmac_tasklet(unsigned long data) 673ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 674ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto int irq; 675ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto u32 csr; 676ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_chan *dc; 677ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 678ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_dev *ddev = (struct txx9dmac_dev *)data; 679ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto u32 mcr; 680ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto int i; 681ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 682ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto mcr = dma_readl(ddev, MCR); 683ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_vdbg(ddev->chan[0]->dma.dev, "tasklet: mcr=%x\n", mcr); 684ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto for (i = 0; i < TXX9_DMA_MAX_NR_CHANNELS; i++) { 685ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if ((mcr >> (24 + i)) & 0x11) { 686ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc = ddev->chan[i]; 687ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto csr = channel_readl(dc, CSR); 688ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_vdbg(chan2dev(&dc->chan), "tasklet: status=%x\n", 689ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto csr); 690ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_lock(&dc->lock); 691ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (csr & (TXX9_DMA_CSR_ABCHC | TXX9_DMA_CSR_NCHNC | 692ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto TXX9_DMA_CSR_NTRNFC)) 693ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_scan_descriptors(dc); 694ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_unlock(&dc->lock); 695ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 696ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 697ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto irq = ddev->irq; 698ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 699ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto enable_irq(irq); 700ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 701ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 702ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic irqreturn_t txx9dmac_interrupt(int irq, void *dev_id) 703ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 704ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_dev *ddev = dev_id; 705ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 706ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_vdbg(ddev->chan[0]->dma.dev, "interrupt: status=%#x\n", 707ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_readl(ddev, MCR)); 708ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 709ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto tasklet_schedule(&ddev->tasklet); 710ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* 711ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * Just disable the interrupts. We'll turn them back on in the 712ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * softirq handler. 713ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto */ 714ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto disable_irq_nosync(irq); 715ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 716ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return IRQ_HANDLED; 717ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 718ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 719ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto/*----------------------------------------------------------------------*/ 720ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 721ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic dma_cookie_t txx9dmac_tx_submit(struct dma_async_tx_descriptor *tx) 722ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 723ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *desc = txd_to_txx9dmac_desc(tx); 724ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_chan *dc = to_txx9dmac_chan(tx->chan); 725ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_cookie_t cookie; 726ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 727ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_lock_bh(&dc->lock); 728884485e1f12dcd39390f042e772cdbefc9ebb750Russell King - ARM Linux cookie = dma_cookie_assign(tx); 729ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 730ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_vdbg(chan2dev(tx->chan), "tx_submit: queued %u %p\n", 731ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->txd.cookie, desc); 732ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 733ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto list_add_tail(&desc->desc_node, &dc->queue); 734ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_unlock_bh(&dc->lock); 735ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 736ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return cookie; 737ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 738ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 739ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic struct dma_async_tx_descriptor * 740ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemototxx9dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 741ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto size_t len, unsigned long flags) 742ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 743ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_chan *dc = to_txx9dmac_chan(chan); 744ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_dev *ddev = dc->ddev; 745ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *desc; 746ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *first; 747ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *prev; 748ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto size_t xfer_count; 749ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto size_t offset; 750ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 751ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_vdbg(chan2dev(chan), "prep_dma_memcpy d%#llx s%#llx l%#zx f%#lx\n", 752ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto (u64)dest, (u64)src, len, flags); 753ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 754ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (unlikely(!len)) { 755ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n"); 756ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return NULL; 757ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 758ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 759ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto prev = first = NULL; 760ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 761ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto for (offset = 0; offset < len; offset += xfer_count) { 762ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto xfer_count = min_t(size_t, len - offset, TXX9_DMA_MAX_COUNT); 763ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* 764ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * Workaround for ERT-TX49H2-033, ERT-TX49H3-020, 765ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * ERT-TX49H4-016 (slightly conservative) 766ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto */ 767ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (__is_dmac64(ddev)) { 768ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (xfer_count > 0x100 && 769ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto (xfer_count & 0xff) >= 0xfa && 770ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto (xfer_count & 0xff) <= 0xff) 771ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto xfer_count -= 0x20; 772ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } else { 773ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (xfer_count > 0x80 && 774ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto (xfer_count & 0x7f) >= 0x7e && 775ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto (xfer_count & 0x7f) <= 0x7f) 776ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto xfer_count -= 0x20; 777ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 778ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 779ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc = txx9dmac_desc_get(dc); 780ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!desc) { 781ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_desc_put(dc, first); 782ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return NULL; 783ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 784ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 785ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (__is_dmac64(ddev)) { 786ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->hwdesc.SAR = src + offset; 787ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->hwdesc.DAR = dest + offset; 788ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->hwdesc.CNTR = xfer_count; 789ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_desc_set_nosimple(ddev, desc, 8, 8, 790ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->ccr | TXX9_DMA_CCR_XFACT); 791ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } else { 792ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->hwdesc32.SAR = src + offset; 793ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->hwdesc32.DAR = dest + offset; 794ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->hwdesc32.CNTR = xfer_count; 795ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_desc_set_nosimple(ddev, desc, 4, 4, 796ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->ccr | TXX9_DMA_CCR_XFACT); 797ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 798ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 799ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* 800ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * The descriptors on tx_list are not reachable from 801ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * the dc->queue list or dc->active_list after a 802ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * submit. If we put all descriptors on active_list, 803ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * calling of callback on the completion will be more 804ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto * complex. 805ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto */ 806ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!first) { 807ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto first = desc; 808ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } else { 809ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc_write_CHAR(dc, prev, desc->txd.phys); 810ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_sync_single_for_device(chan2parent(&dc->chan), 811ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto prev->txd.phys, ddev->descsize, 812ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto DMA_TO_DEVICE); 8131979b186b80449ac6574d97c254b694c8a99b703Dan Williams list_add_tail(&desc->desc_node, &first->tx_list); 814ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 815ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto prev = desc; 816ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 817ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 818ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* Trigger interrupt after last block */ 819ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (flags & DMA_PREP_INTERRUPT) 820ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_desc_set_INTENT(ddev, prev); 821ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 822ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc_write_CHAR(dc, prev, 0); 823ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_sync_single_for_device(chan2parent(&dc->chan), 824ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto prev->txd.phys, ddev->descsize, 825ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto DMA_TO_DEVICE); 826ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 827ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto first->txd.flags = flags; 828ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto first->len = len; 829ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 830ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return &first->txd; 831ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 832ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 833ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic struct dma_async_tx_descriptor * 834ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemototxx9dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 835db8196df4bb6f117caa163aa73b0f16fd62290bdVinod Koul unsigned int sg_len, enum dma_transfer_direction direction, 836185ecb5f4fd43911c35956d4cc7d94a1da30417fAlexandre Bounine unsigned long flags, void *context) 837ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 838ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_chan *dc = to_txx9dmac_chan(chan); 839ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_dev *ddev = dc->ddev; 840ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_slave *ds = chan->private; 841ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *prev; 842ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *first; 843ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto unsigned int i; 844ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct scatterlist *sg; 845ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 846ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_vdbg(chan2dev(chan), "prep_dma_slave\n"); 847ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 848ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto BUG_ON(!ds || !ds->reg_width); 849ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (ds->tx_reg) 850db8196df4bb6f117caa163aa73b0f16fd62290bdVinod Koul BUG_ON(direction != DMA_MEM_TO_DEV); 851ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto else 852db8196df4bb6f117caa163aa73b0f16fd62290bdVinod Koul BUG_ON(direction != DMA_DEV_TO_MEM); 853ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (unlikely(!sg_len)) 854ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return NULL; 855ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 856ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto prev = first = NULL; 857ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 858ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto for_each_sg(sgl, sg, sg_len, i) { 859ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *desc; 860ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_addr_t mem; 861ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto u32 sai, dai; 862ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 863ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc = txx9dmac_desc_get(dc); 864ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!desc) { 865ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_desc_put(dc, first); 866ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return NULL; 867ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 868ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 869ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto mem = sg_dma_address(sg); 870ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 871ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (__is_dmac64(ddev)) { 872db8196df4bb6f117caa163aa73b0f16fd62290bdVinod Koul if (direction == DMA_MEM_TO_DEV) { 873ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->hwdesc.SAR = mem; 874ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->hwdesc.DAR = ds->tx_reg; 875ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } else { 876ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->hwdesc.SAR = ds->rx_reg; 877ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->hwdesc.DAR = mem; 878ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 879ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->hwdesc.CNTR = sg_dma_len(sg); 880ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } else { 881db8196df4bb6f117caa163aa73b0f16fd62290bdVinod Koul if (direction == DMA_MEM_TO_DEV) { 882ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->hwdesc32.SAR = mem; 883ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->hwdesc32.DAR = ds->tx_reg; 884ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } else { 885ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->hwdesc32.SAR = ds->rx_reg; 886ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->hwdesc32.DAR = mem; 887ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 888ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc->hwdesc32.CNTR = sg_dma_len(sg); 889ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 890db8196df4bb6f117caa163aa73b0f16fd62290bdVinod Koul if (direction == DMA_MEM_TO_DEV) { 891ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto sai = ds->reg_width; 892ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dai = 0; 893ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } else { 894ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto sai = 0; 895ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dai = ds->reg_width; 896ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 897ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_desc_set_nosimple(ddev, desc, sai, dai, 898ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->ccr | TXX9_DMA_CCR_XFACT); 899ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 900ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!first) { 901ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto first = desc; 902ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } else { 903ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc_write_CHAR(dc, prev, desc->txd.phys); 904ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_sync_single_for_device(chan2parent(&dc->chan), 905ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto prev->txd.phys, 906ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto ddev->descsize, 907ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto DMA_TO_DEVICE); 9081979b186b80449ac6574d97c254b694c8a99b703Dan Williams list_add_tail(&desc->desc_node, &first->tx_list); 909ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 910ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto prev = desc; 911ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 912ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 913ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* Trigger interrupt after last block */ 914ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (flags & DMA_PREP_INTERRUPT) 915ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_desc_set_INTENT(ddev, prev); 916ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 917ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc_write_CHAR(dc, prev, 0); 918ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_sync_single_for_device(chan2parent(&dc->chan), 919ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto prev->txd.phys, ddev->descsize, 920ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto DMA_TO_DEVICE); 921ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 922ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto first->txd.flags = flags; 923ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto first->len = 0; 924ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 925ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return &first->txd; 926ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 927ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 928058276303dbc4ed089c1f7dad0871810b1f5ddf1Linus Walleijstatic int txx9dmac_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, 929058276303dbc4ed089c1f7dad0871810b1f5ddf1Linus Walleij unsigned long arg) 930ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 931ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_chan *dc = to_txx9dmac_chan(chan); 932ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *desc, *_desc; 933ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto LIST_HEAD(list); 934ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 935c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij /* Only supports DMA_TERMINATE_ALL */ 936c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij if (cmd != DMA_TERMINATE_ALL) 937c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij return -EINVAL; 938c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij 939ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_vdbg(chan2dev(chan), "terminate_all\n"); 940ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_lock_bh(&dc->lock); 941ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 942ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_reset_chan(dc); 943ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 944ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* active_list entries will end up before queued entries */ 945ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto list_splice_init(&dc->queue, &list); 946ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto list_splice_init(&dc->active_list, &list); 947ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 948ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_unlock_bh(&dc->lock); 949ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 950ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* Flush all pending and queued descriptors */ 951ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto list_for_each_entry_safe(desc, _desc, &list, desc_node) 952ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_descriptor_complete(dc, desc); 953c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij 954c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij return 0; 955ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 956ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 957ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic enum dma_status 9580793448187643b50af89d36b08470baf45a3cab4Linus Walleijtxx9dmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, 9590793448187643b50af89d36b08470baf45a3cab4Linus Walleij struct dma_tx_state *txstate) 960ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 961ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_chan *dc = to_txx9dmac_chan(chan); 96296a2af41c78b1fbb1f567a3486bdc63f7b31c5fdRussell King - ARM Linux enum dma_status ret; 963ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 96496a2af41c78b1fbb1f567a3486bdc63f7b31c5fdRussell King - ARM Linux ret = dma_cookie_status(chan, cookie, txstate); 965ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (ret != DMA_SUCCESS) { 966ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_lock_bh(&dc->lock); 967ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_scan_descriptors(dc); 968ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_unlock_bh(&dc->lock); 969ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 97096a2af41c78b1fbb1f567a3486bdc63f7b31c5fdRussell King - ARM Linux ret = dma_cookie_status(chan, cookie, txstate); 971ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 972ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 973ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return ret; 974ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 975ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 976ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void txx9dmac_chain_dynamic(struct txx9dmac_chan *dc, 977ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *prev) 978ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 979ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_dev *ddev = dc->ddev; 980ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *desc; 981ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto LIST_HEAD(list); 982ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 983ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto prev = txx9dmac_last_child(prev); 984ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_dequeue(dc, &list); 985ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc = list_entry(list.next, struct txx9dmac_desc, desc_node); 986ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc_write_CHAR(dc, prev, desc->txd.phys); 987ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_sync_single_for_device(chan2parent(&dc->chan), 988ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto prev->txd.phys, ddev->descsize, 989ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto DMA_TO_DEVICE); 990ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto mmiowb(); 991ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!(channel_readl(dc, CSR) & TXX9_DMA_CSR_CHNEN) && 992ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel_read_CHAR(dc) == prev->txd.phys) 993ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* Restart chain DMA */ 994ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto channel_write_CHAR(dc, desc->txd.phys); 995ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto list_splice_tail(&list, &dc->active_list); 996ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 997ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 998ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void txx9dmac_issue_pending(struct dma_chan *chan) 999ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 1000ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_chan *dc = to_txx9dmac_chan(chan); 1001ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1002ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_lock_bh(&dc->lock); 1003ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1004ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!list_empty(&dc->active_list)) 1005ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_scan_descriptors(dc); 1006ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!list_empty(&dc->queue)) { 1007ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (list_empty(&dc->active_list)) { 1008ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_dequeue(dc, &dc->active_list); 1009ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_dostart(dc, txx9dmac_first_active(dc)); 1010ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } else if (txx9_dma_have_SMPCHN()) { 1011ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *prev = txx9dmac_last_active(dc); 1012ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1013ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!(prev->txd.flags & DMA_PREP_INTERRUPT) || 1014ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_chan_INTENT(dc)) 1015ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_chain_dynamic(dc, prev); 1016ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 1017ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 1018ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1019ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_unlock_bh(&dc->lock); 1020ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 1021ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1022ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic int txx9dmac_alloc_chan_resources(struct dma_chan *chan) 1023ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 1024ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_chan *dc = to_txx9dmac_chan(chan); 1025ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_slave *ds = chan->private; 1026ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *desc; 1027ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto int i; 1028ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1029ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_vdbg(chan2dev(chan), "alloc_chan_resources\n"); 1030ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1031ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* ASSERT: channel is idle */ 1032ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (channel_readl(dc, CSR) & TXX9_DMA_CSR_XFACT) { 1033ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_dbg(chan2dev(chan), "DMA channel not idle?\n"); 1034ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return -EIO; 1035ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 1036ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1037d3ee98cdcd6198ea1cf75c603178acc8a805b69bRussell King - ARM Linux dma_cookie_init(chan); 1038ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1039ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->ccr = TXX9_DMA_CCR_IMMCHN | TXX9_DMA_CCR_INTENE | CCR_LE; 1040ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_chan_set_SMPCHN(dc); 1041ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!txx9_dma_have_SMPCHN() || (dc->ccr & TXX9_DMA_CCR_SMPCHN)) 1042ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->ccr |= TXX9_DMA_CCR_INTENC; 1043ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (chan->device->device_prep_dma_memcpy) { 1044ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (ds) 1045ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return -EINVAL; 1046ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->ccr |= TXX9_DMA_CCR_XFSZ_X8; 1047ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } else { 1048ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!ds || 1049ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto (ds->tx_reg && ds->rx_reg) || (!ds->tx_reg && !ds->rx_reg)) 1050ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return -EINVAL; 1051ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->ccr |= TXX9_DMA_CCR_EXTRQ | 1052ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto TXX9_DMA_CCR_XFSZ(__ffs(ds->reg_width)); 1053ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_chan_set_INTENT(dc); 1054ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 1055ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1056ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_lock_bh(&dc->lock); 1057ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto i = dc->descs_allocated; 1058ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto while (dc->descs_allocated < TXX9_DMA_INITIAL_DESC_COUNT) { 1059ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_unlock_bh(&dc->lock); 1060ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1061ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto desc = txx9dmac_desc_alloc(dc, GFP_KERNEL); 1062ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!desc) { 1063ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_info(chan2dev(chan), 1064ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto "only allocated %d descriptors\n", i); 1065ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_lock_bh(&dc->lock); 1066ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto break; 1067ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 1068ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_desc_put(dc, desc); 1069ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1070ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_lock_bh(&dc->lock); 1071ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto i = ++dc->descs_allocated; 1072ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 1073ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_unlock_bh(&dc->lock); 1074ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1075ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_dbg(chan2dev(chan), 1076ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto "alloc_chan_resources allocated %d descriptors\n", i); 1077ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1078ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return i; 1079ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 1080ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1081ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void txx9dmac_free_chan_resources(struct dma_chan *chan) 1082ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 1083ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_chan *dc = to_txx9dmac_chan(chan); 1084ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_dev *ddev = dc->ddev; 1085ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_desc *desc, *_desc; 1086ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto LIST_HEAD(list); 1087ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1088ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_dbg(chan2dev(chan), "free_chan_resources (descs allocated=%u)\n", 1089ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->descs_allocated); 1090ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1091ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* ASSERT: channel is idle */ 1092ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto BUG_ON(!list_empty(&dc->active_list)); 1093ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto BUG_ON(!list_empty(&dc->queue)); 1094ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto BUG_ON(channel_readl(dc, CSR) & TXX9_DMA_CSR_XFACT); 1095ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1096ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_lock_bh(&dc->lock); 1097ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto list_splice_init(&dc->free_list, &list); 1098ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->descs_allocated = 0; 1099ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_unlock_bh(&dc->lock); 1100ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1101ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto list_for_each_entry_safe(desc, _desc, &list, desc_node) { 1102ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_vdbg(chan2dev(chan), " freeing descriptor %p\n", desc); 1103ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_unmap_single(chan2parent(chan), desc->txd.phys, 1104ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto ddev->descsize, DMA_TO_DEVICE); 1105ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto kfree(desc); 1106ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 1107ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1108ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_vdbg(chan2dev(chan), "free_chan_resources done\n"); 1109ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 1110ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1111ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto/*----------------------------------------------------------------------*/ 1112ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1113ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void txx9dmac_off(struct txx9dmac_dev *ddev) 1114ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 1115ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_writel(ddev, MCR, 0); 1116ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto mmiowb(); 1117ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 1118ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1119ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic int __init txx9dmac_chan_probe(struct platform_device *pdev) 1120ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 1121ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_chan_platform_data *cpdata = pdev->dev.platform_data; 1122ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct platform_device *dmac_dev = cpdata->dmac_dev; 1123ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_platform_data *pdata = dmac_dev->dev.platform_data; 1124ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_chan *dc; 1125ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto int err; 1126ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto int ch = pdev->id % TXX9_DMA_MAX_NR_CHANNELS; 1127ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto int irq; 1128ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1129ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc = devm_kzalloc(&pdev->dev, sizeof(*dc), GFP_KERNEL); 1130ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!dc) 1131ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return -ENOMEM; 1132ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1133ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->dma.dev = &pdev->dev; 1134ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->dma.device_alloc_chan_resources = txx9dmac_alloc_chan_resources; 1135ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->dma.device_free_chan_resources = txx9dmac_free_chan_resources; 1136c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij dc->dma.device_control = txx9dmac_control; 11370793448187643b50af89d36b08470baf45a3cab4Linus Walleij dc->dma.device_tx_status = txx9dmac_tx_status; 1138ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->dma.device_issue_pending = txx9dmac_issue_pending; 1139ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (pdata && pdata->memcpy_chan == ch) { 1140ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->dma.device_prep_dma_memcpy = txx9dmac_prep_dma_memcpy; 1141ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_cap_set(DMA_MEMCPY, dc->dma.cap_mask); 1142ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } else { 1143ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->dma.device_prep_slave_sg = txx9dmac_prep_slave_sg; 1144ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_cap_set(DMA_SLAVE, dc->dma.cap_mask); 1145ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_cap_set(DMA_PRIVATE, dc->dma.cap_mask); 1146ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 1147ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1148ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto INIT_LIST_HEAD(&dc->dma.channels); 1149ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->ddev = platform_get_drvdata(dmac_dev); 1150ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (dc->ddev->irq < 0) { 1151ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto irq = platform_get_irq(pdev, 0); 1152ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (irq < 0) 1153ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return irq; 1154ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto tasklet_init(&dc->tasklet, txx9dmac_chan_tasklet, 1155ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto (unsigned long)dc); 1156ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->irq = irq; 1157ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto err = devm_request_irq(&pdev->dev, dc->irq, 1158ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_chan_interrupt, 0, dev_name(&pdev->dev), dc); 1159ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (err) 1160ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return err; 1161ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } else 1162ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->irq = -1; 1163ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->ddev->chan[ch] = dc; 1164ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->chan.device = &dc->dma; 1165ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto list_add_tail(&dc->chan.device_node, &dc->chan.device->channels); 1166d3ee98cdcd6198ea1cf75c603178acc8a805b69bRussell King - ARM Linux dma_cookie_init(&dc->chan); 1167ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1168ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (is_dmac64(dc)) 1169ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->ch_regs = &__txx9dmac_regs(dc->ddev)->CHAN[ch]; 1170ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto else 1171ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->ch_regs = &__txx9dmac_regs32(dc->ddev)->CHAN[ch]; 1172ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto spin_lock_init(&dc->lock); 1173ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1174ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto INIT_LIST_HEAD(&dc->active_list); 1175ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto INIT_LIST_HEAD(&dc->queue); 1176ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto INIT_LIST_HEAD(&dc->free_list); 1177ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1178ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_reset_chan(dc); 1179ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1180ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto platform_set_drvdata(pdev, dc); 1181ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1182ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto err = dma_async_device_register(&dc->dma); 1183ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (err) 1184ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return err; 1185ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_dbg(&pdev->dev, "TXx9 DMA Channel (dma%d%s%s)\n", 1186ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->dma.dev_id, 1187ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_has_cap(DMA_MEMCPY, dc->dma.cap_mask) ? " memcpy" : "", 1188ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_has_cap(DMA_SLAVE, dc->dma.cap_mask) ? " slave" : ""); 1189ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1190ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return 0; 1191ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 1192ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1193ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic int __exit txx9dmac_chan_remove(struct platform_device *pdev) 1194ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 1195ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_chan *dc = platform_get_drvdata(pdev); 1196ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1197ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_async_device_unregister(&dc->dma); 1198ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (dc->irq >= 0) 1199ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto tasklet_kill(&dc->tasklet); 1200ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dc->ddev->chan[pdev->id % TXX9_DMA_MAX_NR_CHANNELS] = NULL; 1201ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return 0; 1202ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 1203ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1204ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic int __init txx9dmac_probe(struct platform_device *pdev) 1205ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 1206ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_platform_data *pdata = pdev->dev.platform_data; 1207ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct resource *io; 1208ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_dev *ddev; 1209ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto u32 mcr; 1210ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto int err; 1211ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1212ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto io = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1213ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!io) 1214ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return -EINVAL; 1215ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1216ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto ddev = devm_kzalloc(&pdev->dev, sizeof(*ddev), GFP_KERNEL); 1217ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!ddev) 1218ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return -ENOMEM; 1219ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1220ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!devm_request_mem_region(&pdev->dev, io->start, resource_size(io), 1221ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dev_name(&pdev->dev))) 1222ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return -EBUSY; 1223ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1224ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto ddev->regs = devm_ioremap(&pdev->dev, io->start, resource_size(io)); 1225ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!ddev->regs) 1226ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return -ENOMEM; 1227ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto ddev->have_64bit_regs = pdata->have_64bit_regs; 1228ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (__is_dmac64(ddev)) 1229ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto ddev->descsize = sizeof(struct txx9dmac_hwdesc); 1230ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto else 1231ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto ddev->descsize = sizeof(struct txx9dmac_hwdesc32); 1232ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1233ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto /* force dma off, just in case */ 1234ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_off(ddev); 1235ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1236ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto ddev->irq = platform_get_irq(pdev, 0); 1237ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (ddev->irq >= 0) { 1238ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto tasklet_init(&ddev->tasklet, txx9dmac_tasklet, 1239ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto (unsigned long)ddev); 1240ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto err = devm_request_irq(&pdev->dev, ddev->irq, 1241ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_interrupt, 0, dev_name(&pdev->dev), ddev); 1242ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (err) 1243ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return err; 1244ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 1245ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1246ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto mcr = TXX9_DMA_MCR_MSTEN | MCR_LE; 1247ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (pdata && pdata->memcpy_chan >= 0) 1248ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto mcr |= TXX9_DMA_MCR_FIFUM(pdata->memcpy_chan); 1249ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_writel(ddev, MCR, mcr); 1250ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1251ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto platform_set_drvdata(pdev, ddev); 1252ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return 0; 1253ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 1254ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1255ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic int __exit txx9dmac_remove(struct platform_device *pdev) 1256ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 1257ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_dev *ddev = platform_get_drvdata(pdev); 1258ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1259ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_off(ddev); 1260ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (ddev->irq >= 0) 1261ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto tasklet_kill(&ddev->tasklet); 1262ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return 0; 1263ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 1264ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1265ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void txx9dmac_shutdown(struct platform_device *pdev) 1266ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 1267ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_dev *ddev = platform_get_drvdata(pdev); 1268ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1269ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_off(ddev); 1270ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 1271ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 12724aebac2fb9645e897864e12cdb8d1e4aadf5b2a4Magnus Dammstatic int txx9dmac_suspend_noirq(struct device *dev) 1273ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 12744aebac2fb9645e897864e12cdb8d1e4aadf5b2a4Magnus Damm struct platform_device *pdev = to_platform_device(dev); 1275ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_dev *ddev = platform_get_drvdata(pdev); 1276ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1277ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_off(ddev); 1278ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return 0; 1279ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 1280ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 12814aebac2fb9645e897864e12cdb8d1e4aadf5b2a4Magnus Dammstatic int txx9dmac_resume_noirq(struct device *dev) 1282ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 12834aebac2fb9645e897864e12cdb8d1e4aadf5b2a4Magnus Damm struct platform_device *pdev = to_platform_device(dev); 1284ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_dev *ddev = platform_get_drvdata(pdev); 1285ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto struct txx9dmac_platform_data *pdata = pdev->dev.platform_data; 1286ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto u32 mcr; 1287ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1288ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto mcr = TXX9_DMA_MCR_MSTEN | MCR_LE; 1289ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (pdata && pdata->memcpy_chan >= 0) 1290ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto mcr |= TXX9_DMA_MCR_FIFUM(pdata->memcpy_chan); 1291ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto dma_writel(ddev, MCR, mcr); 1292ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return 0; 1293ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1294ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 1295ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1296471452104b8520337ae2fb48c4e61cd4896e025dAlexey Dobriyanstatic const struct dev_pm_ops txx9dmac_dev_pm_ops = { 12974aebac2fb9645e897864e12cdb8d1e4aadf5b2a4Magnus Damm .suspend_noirq = txx9dmac_suspend_noirq, 12984aebac2fb9645e897864e12cdb8d1e4aadf5b2a4Magnus Damm .resume_noirq = txx9dmac_resume_noirq, 12994aebac2fb9645e897864e12cdb8d1e4aadf5b2a4Magnus Damm}; 13004aebac2fb9645e897864e12cdb8d1e4aadf5b2a4Magnus Damm 1301ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic struct platform_driver txx9dmac_chan_driver = { 1302ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto .remove = __exit_p(txx9dmac_chan_remove), 1303ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto .driver = { 1304ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto .name = "txx9dmac-chan", 1305ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto }, 1306ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto}; 1307ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1308ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic struct platform_driver txx9dmac_driver = { 1309ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto .remove = __exit_p(txx9dmac_remove), 1310ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto .shutdown = txx9dmac_shutdown, 1311ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto .driver = { 1312ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto .name = "txx9dmac", 13134aebac2fb9645e897864e12cdb8d1e4aadf5b2a4Magnus Damm .pm = &txx9dmac_dev_pm_ops, 1314ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto }, 1315ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto}; 1316ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1317ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic int __init txx9dmac_init(void) 1318ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 1319ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto int rc; 1320ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1321ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto rc = platform_driver_probe(&txx9dmac_driver, txx9dmac_probe); 1322ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (!rc) { 1323ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto rc = platform_driver_probe(&txx9dmac_chan_driver, 1324ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto txx9dmac_chan_probe); 1325ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto if (rc) 1326ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto platform_driver_unregister(&txx9dmac_driver); 1327ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto } 1328ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto return rc; 1329ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 1330ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotomodule_init(txx9dmac_init); 1331ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1332ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotostatic void __exit txx9dmac_exit(void) 1333ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto{ 1334ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto platform_driver_unregister(&txx9dmac_chan_driver); 1335ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto platform_driver_unregister(&txx9dmac_driver); 1336ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto} 1337ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemotomodule_exit(txx9dmac_exit); 1338ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi Nemoto 1339ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi NemotoMODULE_LICENSE("GPL"); 1340ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi NemotoMODULE_DESCRIPTION("TXx9 DMA Controller driver"); 1341ea76f0b3759283ec3cc06c86e266bf0fa6a981d2Atsushi NemotoMODULE_AUTHOR("Atsushi Nemoto <anemo@mba.ocn.ne.jp>"); 1342b0b4ce38a535ed3de5ec6fdd4f3c34435a1c1d1eGeert UytterhoevenMODULE_ALIAS("platform:txx9dmac"); 1343b0b4ce38a535ed3de5ec6fdd4f3c34435a1c1d1eGeert UytterhoevenMODULE_ALIAS("platform:txx9dmac-chan"); 1344