1d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu/* 2d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * Renesas SuperH DMA Engine support 3d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * 4d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * base is drivers/dma/flsdma.c 5d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * 6d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * Copyright (C) 2009 Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com> 7d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * Copyright (C) 2009 Renesas Solutions, Inc. All rights reserved. 8d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved. 9d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * 10d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * This is free software; you can redistribute it and/or modify 11d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * it under the terms of the GNU General Public License as published by 12d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * the Free Software Foundation; either version 2 of the License, or 13d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * (at your option) any later version. 14d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * 15d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * - DMA of SuperH does not have Hardware DMA chain mode. 16d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * - MAX DMA size is 16MB. 17d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * 18d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu */ 19d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 20d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include <linux/init.h> 21d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include <linux/module.h> 225a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 23d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include <linux/interrupt.h> 24d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include <linux/dmaengine.h> 25d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include <linux/delay.h> 26d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include <linux/platform_device.h> 2720f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski#include <linux/pm_runtime.h> 28b2623a61cfd3c6badb8396dc85ab5a70f4a05f61Magnus Damm#include <linux/sh_dma.h> 2903aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt#include <linux/notifier.h> 3003aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt#include <linux/kdebug.h> 3103aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt#include <linux/spinlock.h> 3203aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt#include <linux/rculist.h> 33d2ebfb335b0426deb1a4fb14e4e926d81ecd8235Russell King - ARM Linux 34d2ebfb335b0426deb1a4fb14e4e926d81ecd8235Russell King - ARM Linux#include "dmaengine.h" 35d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include "shdma.h" 36d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 37d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu/* DMA descriptor control */ 383542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetskienum sh_dmae_desc_status { 393542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski DESC_IDLE, 403542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski DESC_PREPARED, 413542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski DESC_SUBMITTED, 423542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski DESC_COMPLETED, /* completed, have to call callback */ 433542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski DESC_WAITING, /* callback called, waiting for ack / re-submit */ 443542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski}; 45d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 46d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#define NR_DESCS_PER_CHANNEL 32 478b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski/* Default MEMCPY transfer size = 2^2 = 4 bytes */ 488b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski#define LOG2_DEFAULT_XFER_SIZE 2 49d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 5003aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt/* 5103aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt * Used for write-side mutual exclusion for the global device list, 522dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski * read-side synchronization by way of RCU, and per-controller data. 5303aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt */ 5403aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundtstatic DEFINE_SPINLOCK(sh_dmae_lock); 5503aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundtstatic LIST_HEAD(sh_dmae_devices); 5603aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 57cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski/* A bitmask with bits enough for enum sh_dmae_slave_chan_id */ 5802ca5083f60521d09f13224596564a405108bc4cMagnus Dammstatic unsigned long sh_dmae_slave_used[BITS_TO_LONGS(SH_DMA_SLAVE_NUMBER)]; 59cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 603542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetskistatic void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan, bool all); 61c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetskistatic void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan); 62c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski 63c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetskistatic void chclr_write(struct sh_dmae_chan *sh_dc, u32 data) 64c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski{ 65c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski struct sh_dmae_device *shdev = to_sh_dev(sh_dc); 66c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski 67c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski __raw_writel(data, shdev->chan_reg + 68c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski shdev->pdata->channel[sh_dc->id].chclr_offset); 69c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski} 703542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 71d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void sh_dmae_writel(struct sh_dmae_chan *sh_dc, u32 data, u32 reg) 72d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 73027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski __raw_writel(data, sh_dc->base + reg / sizeof(u32)); 74d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 75d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 76d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic u32 sh_dmae_readl(struct sh_dmae_chan *sh_dc, u32 reg) 77d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 78027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski return __raw_readl(sh_dc->base + reg / sizeof(u32)); 79027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski} 80027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 81027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskistatic u16 dmaor_read(struct sh_dmae_device *shdev) 82027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski{ 83e76c3af873025f5a704d56a28882be761e15657bKuninori Morimoto u32 __iomem *addr = shdev->chan_reg + DMAOR / sizeof(u32); 84e76c3af873025f5a704d56a28882be761e15657bKuninori Morimoto 85e76c3af873025f5a704d56a28882be761e15657bKuninori Morimoto if (shdev->pdata->dmaor_is_32bit) 86e76c3af873025f5a704d56a28882be761e15657bKuninori Morimoto return __raw_readl(addr); 87e76c3af873025f5a704d56a28882be761e15657bKuninori Morimoto else 88e76c3af873025f5a704d56a28882be761e15657bKuninori Morimoto return __raw_readw(addr); 89027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski} 90027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 91027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskistatic void dmaor_write(struct sh_dmae_device *shdev, u16 data) 92027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski{ 93e76c3af873025f5a704d56a28882be761e15657bKuninori Morimoto u32 __iomem *addr = shdev->chan_reg + DMAOR / sizeof(u32); 94e76c3af873025f5a704d56a28882be761e15657bKuninori Morimoto 95e76c3af873025f5a704d56a28882be761e15657bKuninori Morimoto if (shdev->pdata->dmaor_is_32bit) 96e76c3af873025f5a704d56a28882be761e15657bKuninori Morimoto __raw_writel(data, addr); 97e76c3af873025f5a704d56a28882be761e15657bKuninori Morimoto else 98e76c3af873025f5a704d56a28882be761e15657bKuninori Morimoto __raw_writew(data, addr); 99d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 100d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1015899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimotostatic void chcr_write(struct sh_dmae_chan *sh_dc, u32 data) 1025899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto{ 1035899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto struct sh_dmae_device *shdev = to_sh_dev(sh_dc); 1045899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto 1055899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto __raw_writel(data, sh_dc->base + shdev->chcr_offset / sizeof(u32)); 1065899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto} 1075899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto 1085899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimotostatic u32 chcr_read(struct sh_dmae_chan *sh_dc) 1095899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto{ 1105899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto struct sh_dmae_device *shdev = to_sh_dev(sh_dc); 1115899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto 1125899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto return __raw_readl(sh_dc->base + shdev->chcr_offset / sizeof(u32)); 113d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 114d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 115d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu/* 116d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * Reset DMA controller 117d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * 118d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * SH7780 has two DMAOR register 119d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu */ 120027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskistatic void sh_dmae_ctl_stop(struct sh_dmae_device *shdev) 121d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1222dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski unsigned short dmaor; 1232dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski unsigned long flags; 1242dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 1252dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_lock_irqsave(&sh_dmae_lock, flags); 126d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1272dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski dmaor = dmaor_read(shdev); 128027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dmaor_write(shdev, dmaor & ~(DMAOR_NMIF | DMAOR_AE | DMAOR_DME)); 1292dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 1302dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_unlock_irqrestore(&sh_dmae_lock, flags); 131d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 132d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 133027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskistatic int sh_dmae_rst(struct sh_dmae_device *shdev) 134d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 135d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu unsigned short dmaor; 1362dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski unsigned long flags; 137d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1382dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_lock_irqsave(&sh_dmae_lock, flags); 139d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1402dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski dmaor = dmaor_read(shdev) & ~(DMAOR_NMIF | DMAOR_AE | DMAOR_DME); 1412dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 142c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski if (shdev->pdata->chclr_present) { 143c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski int i; 144c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski for (i = 0; i < shdev->pdata->channel_num; i++) { 145c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski struct sh_dmae_chan *sh_chan = shdev->chan[i]; 146c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski if (sh_chan) 147c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski chclr_write(sh_chan, 0); 148c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski } 149c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski } 150c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski 1512dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski dmaor_write(shdev, dmaor | shdev->pdata->dmaor_init); 1522dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 1532dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski dmaor = dmaor_read(shdev); 1542dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 1552dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_unlock_irqrestore(&sh_dmae_lock, flags); 1562dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 1572dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski if (dmaor & (DMAOR_AE | DMAOR_NMIF)) { 1582dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski dev_warn(shdev->common.dev, "Can't initialize DMAOR.\n"); 1592dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski return -EIO; 160d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 161c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski if (shdev->pdata->dmaor_init & ~dmaor) 162c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski dev_warn(shdev->common.dev, 163c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski "DMAOR=0x%x hasn't latched the initial value 0x%x.\n", 164c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski dmaor, shdev->pdata->dmaor_init); 165d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return 0; 166d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 167d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 168fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetskistatic bool dmae_is_busy(struct sh_dmae_chan *sh_chan) 169d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1705899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto u32 chcr = chcr_read(sh_chan); 171fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 172fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if ((chcr & (CHCR_DE | CHCR_TE)) == CHCR_DE) 173fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return true; /* working */ 174fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 175fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return false; /* waiting */ 176d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 177d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1788b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetskistatic unsigned int calc_xmit_shift(struct sh_dmae_chan *sh_chan, u32 chcr) 179d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 180c4e0dd7835d12d9765a372b586a5020ac29cc706Kuninori Morimoto struct sh_dmae_device *shdev = to_sh_dev(sh_chan); 1818b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski struct sh_dmae_pdata *pdata = shdev->pdata; 1828b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski int cnt = ((chcr & pdata->ts_low_mask) >> pdata->ts_low_shift) | 1838b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski ((chcr & pdata->ts_high_mask) >> pdata->ts_high_shift); 1848b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski 1858b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski if (cnt >= pdata->ts_shift_num) 1868b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski cnt = 0; 187623b4ac4bf9e767991c66e29b47dd4b19458fb42Guennadi Liakhovetski 1888b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski return pdata->ts_shift[cnt]; 1898b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski} 1908b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski 1918b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetskistatic u32 log2size_to_chcr(struct sh_dmae_chan *sh_chan, int l2size) 1928b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski{ 193c4e0dd7835d12d9765a372b586a5020ac29cc706Kuninori Morimoto struct sh_dmae_device *shdev = to_sh_dev(sh_chan); 1948b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski struct sh_dmae_pdata *pdata = shdev->pdata; 1958b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski int i; 1968b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski 1978b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski for (i = 0; i < pdata->ts_shift_num; i++) 1988b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski if (pdata->ts_shift[i] == l2size) 1998b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski break; 2008b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski 2018b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski if (i == pdata->ts_shift_num) 2028b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski i = 0; 2038b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski 2048b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski return ((i << pdata->ts_low_shift) & pdata->ts_low_mask) | 2058b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski ((i << pdata->ts_high_shift) & pdata->ts_high_mask); 206d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 207d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 2083542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetskistatic void dmae_set_reg(struct sh_dmae_chan *sh_chan, struct sh_dmae_regs *hw) 209d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 2103542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski sh_dmae_writel(sh_chan, hw->sar, SAR); 2113542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski sh_dmae_writel(sh_chan, hw->dar, DAR); 212cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski sh_dmae_writel(sh_chan, hw->tcr >> sh_chan->xmit_shift, TCR); 213d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 214d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 215d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void dmae_start(struct sh_dmae_chan *sh_chan) 216d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 21767c6269e5c998b53c2c08ce2befbbe20a7b6f57fKuninori Morimoto struct sh_dmae_device *shdev = to_sh_dev(sh_chan); 2185899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto u32 chcr = chcr_read(sh_chan); 219d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 220260bf2c5f69f419b04b6861ca91565b5fb16ce48Kuninori Morimoto if (shdev->pdata->needs_tend_set) 221260bf2c5f69f419b04b6861ca91565b5fb16ce48Kuninori Morimoto sh_dmae_writel(sh_chan, 0xFFFFFFFF, TEND); 222260bf2c5f69f419b04b6861ca91565b5fb16ce48Kuninori Morimoto 22367c6269e5c998b53c2c08ce2befbbe20a7b6f57fKuninori Morimoto chcr |= CHCR_DE | shdev->chcr_ie_bit; 2245899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto chcr_write(sh_chan, chcr & ~CHCR_TE); 225d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 226d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 227d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void dmae_halt(struct sh_dmae_chan *sh_chan) 228d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 22967c6269e5c998b53c2c08ce2befbbe20a7b6f57fKuninori Morimoto struct sh_dmae_device *shdev = to_sh_dev(sh_chan); 2305899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto u32 chcr = chcr_read(sh_chan); 231d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 23267c6269e5c998b53c2c08ce2befbbe20a7b6f57fKuninori Morimoto chcr &= ~(CHCR_DE | CHCR_TE | shdev->chcr_ie_bit); 2335899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto chcr_write(sh_chan, chcr); 234d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 235d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 236cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetskistatic void dmae_init(struct sh_dmae_chan *sh_chan) 237cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski{ 2388b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski /* 2398b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski * Default configuration for dual address memory-memory transfer. 2408b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski * 0x400 represents auto-request. 2418b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski */ 2428b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski u32 chcr = DM_INC | SM_INC | 0x400 | log2size_to_chcr(sh_chan, 2438b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski LOG2_DEFAULT_XFER_SIZE); 2448b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski sh_chan->xmit_shift = calc_xmit_shift(sh_chan, chcr); 2455899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto chcr_write(sh_chan, chcr); 246cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski} 247cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 248d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic int dmae_set_chcr(struct sh_dmae_chan *sh_chan, u32 val) 249d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 2502dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski /* If DMA is active, cannot set CHCR. TODO: remove this superfluous check */ 251fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (dmae_is_busy(sh_chan)) 252fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return -EBUSY; 253d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 2548b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski sh_chan->xmit_shift = calc_xmit_shift(sh_chan, val); 2555899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto chcr_write(sh_chan, val); 256cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 257d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return 0; 258d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 259d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 260d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic int dmae_set_dmars(struct sh_dmae_chan *sh_chan, u16 val) 261d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 262c4e0dd7835d12d9765a372b586a5020ac29cc706Kuninori Morimoto struct sh_dmae_device *shdev = to_sh_dev(sh_chan); 263027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct sh_dmae_pdata *pdata = shdev->pdata; 2645bac942db3d2c4738df04104240d65a5d1eaec6aGuennadi Liakhovetski const struct sh_dmae_channel *chan_pdata = &pdata->channel[sh_chan->id]; 26526fc02ab5551349b2e593829a76cb44328ee7f61Magnus Damm u16 __iomem *addr = shdev->dmars; 266090b91805a97f58a7deff0f2b40aad1cc2f1b7e0Kuninori Morimoto unsigned int shift = chan_pdata->dmars_bit; 267fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 268fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (dmae_is_busy(sh_chan)) 269fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return -EBUSY; 270d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 271260bf2c5f69f419b04b6861ca91565b5fb16ce48Kuninori Morimoto if (pdata->no_dmars) 272260bf2c5f69f419b04b6861ca91565b5fb16ce48Kuninori Morimoto return 0; 273260bf2c5f69f419b04b6861ca91565b5fb16ce48Kuninori Morimoto 27426fc02ab5551349b2e593829a76cb44328ee7f61Magnus Damm /* in the case of a missing DMARS resource use first memory window */ 27526fc02ab5551349b2e593829a76cb44328ee7f61Magnus Damm if (!addr) 27626fc02ab5551349b2e593829a76cb44328ee7f61Magnus Damm addr = (u16 __iomem *)shdev->chan_reg; 27726fc02ab5551349b2e593829a76cb44328ee7f61Magnus Damm addr += chan_pdata->dmars / sizeof(u16); 27826fc02ab5551349b2e593829a76cb44328ee7f61Magnus Damm 279027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski __raw_writew((__raw_readw(addr) & (0xff00 >> shift)) | (val << shift), 280027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski addr); 281d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 282d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return 0; 283d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 284d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 285d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic dma_cookie_t sh_dmae_tx_submit(struct dma_async_tx_descriptor *tx) 286d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 2873542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski struct sh_desc *desc = tx_to_sh_desc(tx), *chunk, *last = desc, *c; 288d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_chan *sh_chan = to_sh_chan(tx->chan); 2897a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski struct sh_dmae_slave *param = tx->chan->private; 2903542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dma_async_tx_callback callback = tx->callback; 291d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_cookie_t cookie; 2927a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski bool power_up; 293d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 2947a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski spin_lock_irq(&sh_chan->desc_lock); 2957a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski 2967a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski if (list_empty(&sh_chan->ld_queue)) 2977a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski power_up = true; 2987a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski else 2997a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski power_up = false; 300d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 301884485e1f12dcd39390f042e772cdbefc9ebb750Russell King - ARM Linux cookie = dma_cookie_assign(tx); 3023542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 3033542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Mark all chunks of this descriptor as submitted, move to the queue */ 3043542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski list_for_each_entry_safe(chunk, c, desc->node.prev, node) { 3053542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* 3063542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * All chunks are on the global ld_free, so, we have to find 3073542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * the end of the chain ourselves 3083542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski */ 3093542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (chunk != desc && (chunk->mark == DESC_IDLE || 3103542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski chunk->async_tx.cookie > 0 || 3113542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski chunk->async_tx.cookie == -EBUSY || 3123542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski &chunk->node == &sh_chan->ld_free)) 3133542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski break; 3143542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski chunk->mark = DESC_SUBMITTED; 3153542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Callback goes to the last chunk */ 3163542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski chunk->async_tx.callback = NULL; 3173542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski chunk->cookie = cookie; 3183542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski list_move_tail(&chunk->node, &sh_chan->ld_queue); 3193542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski last = chunk; 3203542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 321d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 3223542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski last->async_tx.callback = callback; 3233542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski last->async_tx.callback_param = tx->callback_param; 3243542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 3253542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dev_dbg(sh_chan->dev, "submit #%d@%p on %d: %x[%d] -> %x\n", 3263542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski tx->cookie, &last->async_tx, sh_chan->id, 3273542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->hw.sar, desc->hw.tcr, desc->hw.dar); 328d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 3297a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski if (power_up) { 3307a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski sh_chan->pm_state = DMAE_PM_BUSY; 3317a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski 3327a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski pm_runtime_get(sh_chan->dev); 3337a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski 3347a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski spin_unlock_irq(&sh_chan->desc_lock); 3357a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski 3367a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski pm_runtime_barrier(sh_chan->dev); 3377a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski 3387a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski spin_lock_irq(&sh_chan->desc_lock); 3397a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski 3407a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski /* Have we been reset, while waiting? */ 3417a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski if (sh_chan->pm_state != DMAE_PM_ESTABLISHED) { 3427a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski dev_dbg(sh_chan->dev, "Bring up channel %d\n", 3437a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski sh_chan->id); 3447a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski if (param) { 3457a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski const struct sh_dmae_slave_config *cfg = 3467a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski param->config; 3477a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski 3487a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski dmae_set_dmars(sh_chan, cfg->mid_rid); 3497a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski dmae_set_chcr(sh_chan, cfg->chcr); 3507a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski } else { 3517a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski dmae_init(sh_chan); 3527a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski } 3537a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski 3547a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski if (sh_chan->pm_state == DMAE_PM_PENDING) 3557a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski sh_chan_xfer_ld_queue(sh_chan); 3567a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski sh_chan->pm_state = DMAE_PM_ESTABLISHED; 3577a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski } 358c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski } else { 359c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski sh_chan->pm_state = DMAE_PM_PENDING; 3607a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski } 3617a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski 3627a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski spin_unlock_irq(&sh_chan->desc_lock); 363d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 364d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return cookie; 365d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 366d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 3673542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski/* Called with desc_lock held */ 368d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic struct sh_desc *sh_dmae_get_desc(struct sh_dmae_chan *sh_chan) 369d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 3703542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski struct sh_desc *desc; 371d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 3723542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski list_for_each_entry(desc, &sh_chan->ld_free, node) 3733542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (desc->mark != DESC_PREPARED) { 3743542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski BUG_ON(desc->mark != DESC_IDLE); 375d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu list_del(&desc->node); 3763542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski return desc; 377d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 378d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 3793542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski return NULL; 380d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 381d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 3825bac942db3d2c4738df04104240d65a5d1eaec6aGuennadi Liakhovetskistatic const struct sh_dmae_slave_config *sh_dmae_find_slave( 3834bab9d426e6dbd9ea09330919a33d35d5faab400Magnus Damm struct sh_dmae_chan *sh_chan, struct sh_dmae_slave *param) 384cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski{ 385c4e0dd7835d12d9765a372b586a5020ac29cc706Kuninori Morimoto struct sh_dmae_device *shdev = to_sh_dev(sh_chan); 386027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct sh_dmae_pdata *pdata = shdev->pdata; 387cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski int i; 388cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 38902ca5083f60521d09f13224596564a405108bc4cMagnus Damm if (param->slave_id >= SH_DMA_SLAVE_NUMBER) 390cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski return NULL; 391cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 392027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski for (i = 0; i < pdata->slave_num; i++) 3934bab9d426e6dbd9ea09330919a33d35d5faab400Magnus Damm if (pdata->slave[i].slave_id == param->slave_id) 394027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski return pdata->slave + i; 395cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 396cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski return NULL; 397cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski} 398cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 399d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic int sh_dmae_alloc_chan_resources(struct dma_chan *chan) 400d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 401d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_chan *sh_chan = to_sh_chan(chan); 402d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_desc *desc; 403cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct sh_dmae_slave *param = chan->private; 40483515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski int ret; 405cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 406cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski /* 407cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski * This relies on the guarantee from dmaengine that alloc_chan_resources 408cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski * never runs concurrently with itself or free_chan_resources. 409cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski */ 410cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski if (param) { 4115bac942db3d2c4738df04104240d65a5d1eaec6aGuennadi Liakhovetski const struct sh_dmae_slave_config *cfg; 412cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 4134bab9d426e6dbd9ea09330919a33d35d5faab400Magnus Damm cfg = sh_dmae_find_slave(sh_chan, param); 41483515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski if (!cfg) { 41583515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski ret = -EINVAL; 41683515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski goto efindslave; 41783515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski } 418cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 41983515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski if (test_and_set_bit(param->slave_id, sh_dmae_slave_used)) { 42083515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski ret = -EBUSY; 42183515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski goto etestused; 42283515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski } 423cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 424cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski param->config = cfg; 425cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski } 426d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 427d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu while (sh_chan->descs_allocated < NR_DESCS_PER_CHANNEL) { 428d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu desc = kzalloc(sizeof(struct sh_desc), GFP_KERNEL); 429b4dae6e1adaedc9c343b5f00332312d649600bdcGuennadi Liakhovetski if (!desc) 430d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu break; 431d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_async_tx_descriptor_init(&desc->async_tx, 432d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu &sh_chan->common); 433d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu desc->async_tx.tx_submit = sh_dmae_tx_submit; 4343542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark = DESC_IDLE; 435d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 4363542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski list_add(&desc->node, &sh_chan->ld_free); 437d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu sh_chan->descs_allocated++; 438d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 439d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 44083515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski if (!sh_chan->descs_allocated) { 44183515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski ret = -ENOMEM; 44283515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski goto edescalloc; 44383515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski } 44420f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski 445d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return sh_chan->descs_allocated; 44683515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski 44783515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetskiedescalloc: 44883515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski if (param) 44983515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski clear_bit(param->slave_id, sh_dmae_slave_used); 45083515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetskietestused: 45183515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetskiefindslave: 452b4dae6e1adaedc9c343b5f00332312d649600bdcGuennadi Liakhovetski chan->private = NULL; 45383515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski return ret; 454d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 455d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 456d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu/* 457d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * sh_dma_free_chan_resources - Free all resources of the channel. 458d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu */ 459d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void sh_dmae_free_chan_resources(struct dma_chan *chan) 460d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 461d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_chan *sh_chan = to_sh_chan(chan); 462d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_desc *desc, *_desc; 463d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu LIST_HEAD(list); 464d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 4652dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski /* Protect against ISR */ 4662dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_lock_irq(&sh_chan->desc_lock); 467cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski dmae_halt(sh_chan); 4682dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_unlock_irq(&sh_chan->desc_lock); 4692dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 4702dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski /* Now no new interrupts will occur */ 471cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 4723542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Prepared and not submitted descriptors can still be on the queue */ 4733542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (!list_empty(&sh_chan->ld_queue)) 4743542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski sh_dmae_chan_ld_cleanup(sh_chan, true); 4753542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 476cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski if (chan->private) { 477cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski /* The caller is holding dma_list_mutex */ 478cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct sh_dmae_slave *param = chan->private; 479cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski clear_bit(param->slave_id, sh_dmae_slave_used); 4802dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski chan->private = NULL; 481cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski } 482cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 483b4dae6e1adaedc9c343b5f00332312d649600bdcGuennadi Liakhovetski spin_lock_irq(&sh_chan->desc_lock); 484d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 485d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu list_splice_init(&sh_chan->ld_free, &list); 486d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu sh_chan->descs_allocated = 0; 487d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 488b4dae6e1adaedc9c343b5f00332312d649600bdcGuennadi Liakhovetski spin_unlock_irq(&sh_chan->desc_lock); 489d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 490d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu list_for_each_entry_safe(desc, _desc, &list, node) 491d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu kfree(desc); 492d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 493d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 494cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski/** 495fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * sh_dmae_add_desc - get, set up and return one transfer descriptor 496fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * @sh_chan: DMA channel 497fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * @flags: DMA transfer flags 498fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * @dest: destination DMA address, incremented when direction equals 499db8196df4bb6f117caa163aa73b0f16fd62290bdVinod Koul * DMA_DEV_TO_MEM 500fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * @src: source DMA address, incremented when direction equals 501db8196df4bb6f117caa163aa73b0f16fd62290bdVinod Koul * DMA_MEM_TO_DEV 502fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * @len: DMA transfer length 503fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * @first: if NULL, set to the current descriptor and cookie set to -EBUSY 504fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * @direction: needed for slave DMA to decide which address to keep constant, 505db8196df4bb6f117caa163aa73b0f16fd62290bdVinod Koul * equals DMA_MEM_TO_MEM for MEMCPY 506fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * Returns 0 or an error 507fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * Locks: called with desc_lock held 508fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski */ 509fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetskistatic struct sh_desc *sh_dmae_add_desc(struct sh_dmae_chan *sh_chan, 510fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski unsigned long flags, dma_addr_t *dest, dma_addr_t *src, size_t *len, 511db8196df4bb6f117caa163aa73b0f16fd62290bdVinod Koul struct sh_desc **first, enum dma_transfer_direction direction) 512d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 513fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct sh_desc *new; 514d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu size_t copy_size; 515d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 516fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (!*len) 517d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return NULL; 518d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 519fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski /* Allocate the link descriptor from the free list */ 520fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new = sh_dmae_get_desc(sh_chan); 521fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (!new) { 522fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski dev_err(sh_chan->dev, "No free link descriptor available\n"); 523d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return NULL; 524fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski } 525d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 526fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski copy_size = min(*len, (size_t)SH_DMA_TCR_MAX + 1); 527fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 528fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->hw.sar = *src; 529fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->hw.dar = *dest; 530fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->hw.tcr = copy_size; 531fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 532fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (!*first) { 533fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski /* First desc */ 534fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->async_tx.cookie = -EBUSY; 535fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski *first = new; 536fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski } else { 537fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski /* Other desc - invisible to the user */ 538fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->async_tx.cookie = -EINVAL; 539fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski } 540fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 541cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski dev_dbg(sh_chan->dev, 542cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski "chaining (%u/%u)@%x -> %x with %p, cookie %d, shift %d\n", 543fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski copy_size, *len, *src, *dest, &new->async_tx, 544cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski new->async_tx.cookie, sh_chan->xmit_shift); 545fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 546fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->mark = DESC_PREPARED; 547fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->async_tx.flags = flags; 548cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski new->direction = direction; 549fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 550fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski *len -= copy_size; 551db8196df4bb6f117caa163aa73b0f16fd62290bdVinod Koul if (direction == DMA_MEM_TO_MEM || direction == DMA_MEM_TO_DEV) 552fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski *src += copy_size; 553db8196df4bb6f117caa163aa73b0f16fd62290bdVinod Koul if (direction == DMA_MEM_TO_MEM || direction == DMA_DEV_TO_MEM) 554fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski *dest += copy_size; 555fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 556fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return new; 557fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski} 558fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 559fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski/* 560fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * sh_dmae_prep_sg - prepare transfer descriptors from an SG list 561fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * 562fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * Common routine for public (MEMCPY) and slave DMA. The MEMCPY case is also 563fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * converted to scatter-gather to guarantee consistent locking and a correct 564fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * list manipulation. For slave DMA direction carries the usual meaning, and, 565fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * logically, the SG list is RAM and the addr variable contains slave address, 566db8196df4bb6f117caa163aa73b0f16fd62290bdVinod Koul * e.g., the FIFO I/O register. For MEMCPY direction equals DMA_MEM_TO_MEM 567fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * and the SG list contains only one element and points at the source buffer. 568fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski */ 569fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetskistatic struct dma_async_tx_descriptor *sh_dmae_prep_sg(struct sh_dmae_chan *sh_chan, 570fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct scatterlist *sgl, unsigned int sg_len, dma_addr_t *addr, 571db8196df4bb6f117caa163aa73b0f16fd62290bdVinod Koul enum dma_transfer_direction direction, unsigned long flags) 572fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski{ 573fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct scatterlist *sg; 574fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct sh_desc *first = NULL, *new = NULL /* compiler... */; 575fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski LIST_HEAD(tx_list); 576fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski int chunks = 0; 577b4dae6e1adaedc9c343b5f00332312d649600bdcGuennadi Liakhovetski unsigned long irq_flags; 578fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski int i; 579fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 580fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (!sg_len) 581fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return NULL; 582fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 583fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski for_each_sg(sgl, sg, sg_len, i) 584fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski chunks += (sg_dma_len(sg) + SH_DMA_TCR_MAX) / 585fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski (SH_DMA_TCR_MAX + 1); 586d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 5873542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Have to lock the whole loop to protect against concurrent release */ 588b4dae6e1adaedc9c343b5f00332312d649600bdcGuennadi Liakhovetski spin_lock_irqsave(&sh_chan->desc_lock, irq_flags); 5893542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 5903542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* 5913542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * Chaining: 5923542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * first descriptor is what user is dealing with in all API calls, its 5933542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * cookie is at first set to -EBUSY, at tx-submit to a positive 5943542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * number 5953542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * if more than one chunk is needed further chunks have cookie = -EINVAL 5963542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * the last chunk, if not equal to the first, has cookie = -ENOSPC 5973542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * all chunks are linked onto the tx_list head with their .node heads 5983542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * only during this function, then they are immediately spliced 5993542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * back onto the free list in form of a chain 6003542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski */ 601fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski for_each_sg(sgl, sg, sg_len, i) { 602fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski dma_addr_t sg_addr = sg_dma_address(sg); 603fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski size_t len = sg_dma_len(sg); 604fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 605fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (!len) 606fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski goto err_get_desc; 607fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 608fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski do { 609fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski dev_dbg(sh_chan->dev, "Add SG #%d@%p[%d], dma %llx\n", 610fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski i, sg, len, (unsigned long long)sg_addr); 611fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 612db8196df4bb6f117caa163aa73b0f16fd62290bdVinod Koul if (direction == DMA_DEV_TO_MEM) 613fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new = sh_dmae_add_desc(sh_chan, flags, 614fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski &sg_addr, addr, &len, &first, 615fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski direction); 616fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski else 617fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new = sh_dmae_add_desc(sh_chan, flags, 618fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski addr, &sg_addr, &len, &first, 619fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski direction); 620fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (!new) 621fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski goto err_get_desc; 622fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 623fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->chunks = chunks--; 624fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski list_add_tail(&new->node, &tx_list); 625fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski } while (len); 626fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski } 627d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 6283542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (new != first) 6293542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski new->async_tx.cookie = -ENOSPC; 630d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 6313542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Put them back on the free list, so, they don't get lost */ 6323542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski list_splice_tail(&tx_list, &sh_chan->ld_free); 633d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 634b4dae6e1adaedc9c343b5f00332312d649600bdcGuennadi Liakhovetski spin_unlock_irqrestore(&sh_chan->desc_lock, irq_flags); 635d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 6363542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski return &first->async_tx; 637fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 638fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetskierr_get_desc: 639fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski list_for_each_entry(new, &tx_list, node) 640fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->mark = DESC_IDLE; 641fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski list_splice(&tx_list, &sh_chan->ld_free); 642fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 643b4dae6e1adaedc9c343b5f00332312d649600bdcGuennadi Liakhovetski spin_unlock_irqrestore(&sh_chan->desc_lock, irq_flags); 644fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 645fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return NULL; 646fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski} 647fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 648fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetskistatic struct dma_async_tx_descriptor *sh_dmae_prep_memcpy( 649fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct dma_chan *chan, dma_addr_t dma_dest, dma_addr_t dma_src, 650fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski size_t len, unsigned long flags) 651fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski{ 652fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct sh_dmae_chan *sh_chan; 653fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct scatterlist sg; 654fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 655fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (!chan || !len) 656fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return NULL; 657fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 658fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski sh_chan = to_sh_chan(chan); 659fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 660fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski sg_init_table(&sg, 1); 661fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski sg_set_page(&sg, pfn_to_page(PFN_DOWN(dma_src)), len, 662fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski offset_in_page(dma_src)); 663fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski sg_dma_address(&sg) = dma_src; 664fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski sg_dma_len(&sg) = len; 665fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 666db8196df4bb6f117caa163aa73b0f16fd62290bdVinod Koul return sh_dmae_prep_sg(sh_chan, &sg, 1, &dma_dest, DMA_MEM_TO_MEM, 667fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski flags); 668d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 669d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 670cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetskistatic struct dma_async_tx_descriptor *sh_dmae_prep_slave_sg( 671cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, 672185ecb5f4fd43911c35956d4cc7d94a1da30417fAlexandre Bounine enum dma_transfer_direction direction, unsigned long flags, 673185ecb5f4fd43911c35956d4cc7d94a1da30417fAlexandre Bounine void *context) 674cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski{ 675cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct sh_dmae_slave *param; 676cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct sh_dmae_chan *sh_chan; 6775bac942db3d2c4738df04104240d65a5d1eaec6aGuennadi Liakhovetski dma_addr_t slave_addr; 678cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 679cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski if (!chan) 680cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski return NULL; 681cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 682cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski sh_chan = to_sh_chan(chan); 683cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski param = chan->private; 684cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 685cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski /* Someone calling slave DMA on a public channel? */ 686cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski if (!param || !sg_len) { 687cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski dev_warn(sh_chan->dev, "%s: bad parameter: %p, %d, %d\n", 688cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski __func__, param, sg_len, param ? param->slave_id : -1); 689cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski return NULL; 690cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski } 691cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 6929f9ff20d46c6728b092f34b6a642e1e81ab5e254Dan Carpenter slave_addr = param->config->addr; 6939f9ff20d46c6728b092f34b6a642e1e81ab5e254Dan Carpenter 694cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski /* 695cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski * if (param != NULL), this is a successfully requested slave channel, 696cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski * therefore param->config != NULL too. 697cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski */ 6985bac942db3d2c4738df04104240d65a5d1eaec6aGuennadi Liakhovetski return sh_dmae_prep_sg(sh_chan, sgl, sg_len, &slave_addr, 699cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski direction, flags); 700cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski} 701cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 702058276303dbc4ed089c1f7dad0871810b1f5ddf1Linus Walleijstatic int sh_dmae_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, 703058276303dbc4ed089c1f7dad0871810b1f5ddf1Linus Walleij unsigned long arg) 704cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski{ 705cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct sh_dmae_chan *sh_chan = to_sh_chan(chan); 706b4dae6e1adaedc9c343b5f00332312d649600bdcGuennadi Liakhovetski unsigned long flags; 707cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 708c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij /* Only supports DMA_TERMINATE_ALL */ 709c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij if (cmd != DMA_TERMINATE_ALL) 710c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij return -ENXIO; 711c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij 712cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski if (!chan) 713c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij return -EINVAL; 714cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 715b4dae6e1adaedc9c343b5f00332312d649600bdcGuennadi Liakhovetski spin_lock_irqsave(&sh_chan->desc_lock, flags); 716c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski dmae_halt(sh_chan); 717c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski 718c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski if (!list_empty(&sh_chan->ld_queue)) { 719c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski /* Record partial transfer */ 720c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski struct sh_desc *desc = list_entry(sh_chan->ld_queue.next, 721c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski struct sh_desc, node); 722c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski desc->partial = (desc->hw.tcr - sh_dmae_readl(sh_chan, TCR)) << 723c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski sh_chan->xmit_shift; 724c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski } 725b4dae6e1adaedc9c343b5f00332312d649600bdcGuennadi Liakhovetski spin_unlock_irqrestore(&sh_chan->desc_lock, flags); 726c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski 727cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski sh_dmae_chan_ld_cleanup(sh_chan, true); 728c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij 729c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij return 0; 730cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski} 731cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 7323542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetskistatic dma_async_tx_callback __ld_cleanup(struct sh_dmae_chan *sh_chan, bool all) 733d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 734d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_desc *desc, *_desc; 7353542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Is the "exposed" head of a chain acked? */ 7363542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski bool head_acked = false; 7373542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dma_cookie_t cookie = 0; 7383542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dma_async_tx_callback callback = NULL; 7393542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski void *param = NULL; 740b4dae6e1adaedc9c343b5f00332312d649600bdcGuennadi Liakhovetski unsigned long flags; 741d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 742b4dae6e1adaedc9c343b5f00332312d649600bdcGuennadi Liakhovetski spin_lock_irqsave(&sh_chan->desc_lock, flags); 743d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu list_for_each_entry_safe(desc, _desc, &sh_chan->ld_queue, node) { 7443542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski struct dma_async_tx_descriptor *tx = &desc->async_tx; 7453542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 7463542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski BUG_ON(tx->cookie > 0 && tx->cookie != desc->cookie); 7473542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski BUG_ON(desc->mark != DESC_SUBMITTED && 7483542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark != DESC_COMPLETED && 7493542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark != DESC_WAITING); 7503542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 7513542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* 7523542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * queue is ordered, and we use this loop to (1) clean up all 7533542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * completed descriptors, and to (2) update descriptor flags of 7543542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * any chunks in a (partially) completed chain 7553542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski */ 7563542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (!all && desc->mark == DESC_SUBMITTED && 7573542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->cookie != cookie) 758d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu break; 759d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 7603542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (tx->cookie > 0) 7613542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski cookie = tx->cookie; 762d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 7633542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (desc->mark == DESC_COMPLETED && desc->chunks == 1) { 7644d4e58de32a192fea65ab84509d17d199bd291c8Russell King - ARM Linux if (sh_chan->common.completed_cookie != desc->cookie - 1) 765cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski dev_dbg(sh_chan->dev, 766cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski "Completing cookie %d, expected %d\n", 767cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski desc->cookie, 7684d4e58de32a192fea65ab84509d17d199bd291c8Russell King - ARM Linux sh_chan->common.completed_cookie + 1); 7694d4e58de32a192fea65ab84509d17d199bd291c8Russell King - ARM Linux sh_chan->common.completed_cookie = desc->cookie; 7703542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 771d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 7723542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Call callback on the last chunk */ 7733542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (desc->mark == DESC_COMPLETED && tx->callback) { 7743542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark = DESC_WAITING; 7753542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski callback = tx->callback; 7763542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski param = tx->callback_param; 7773542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dev_dbg(sh_chan->dev, "descriptor #%d@%p on %d callback\n", 7783542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski tx->cookie, tx, sh_chan->id); 7793542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski BUG_ON(desc->chunks != 1); 7803542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski break; 7813542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 782d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 7833542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (tx->cookie > 0 || tx->cookie == -EBUSY) { 7843542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (desc->mark == DESC_COMPLETED) { 7853542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski BUG_ON(tx->cookie < 0); 7863542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark = DESC_WAITING; 7873542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 7883542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski head_acked = async_tx_test_ack(tx); 7893542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } else { 7903542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski switch (desc->mark) { 7913542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski case DESC_COMPLETED: 7923542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark = DESC_WAITING; 7933542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Fall through */ 7943542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski case DESC_WAITING: 7953542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (head_acked) 7963542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski async_tx_ack(&desc->async_tx); 7973542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 7983542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 7993542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 8003542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dev_dbg(sh_chan->dev, "descriptor %p #%d completed.\n", 8013542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski tx, tx->cookie); 8023542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 8033542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (((desc->mark == DESC_COMPLETED || 8043542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark == DESC_WAITING) && 8053542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski async_tx_test_ack(&desc->async_tx)) || all) { 8063542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Remove from ld_queue list */ 8073542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark = DESC_IDLE; 8087a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski 8093542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski list_move(&desc->node, &sh_chan->ld_free); 8107a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski 8117a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski if (list_empty(&sh_chan->ld_queue)) { 8127a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski dev_dbg(sh_chan->dev, "Bring down channel %d\n", sh_chan->id); 8137a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski pm_runtime_put(sh_chan->dev); 8147a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski } 815d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 816d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 8172dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 8182dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski if (all && !callback) 8192dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski /* 8202dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski * Terminating and the loop completed normally: forgive 8212dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski * uncompleted cookies 8222dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski */ 8234d4e58de32a192fea65ab84509d17d199bd291c8Russell King - ARM Linux sh_chan->common.completed_cookie = sh_chan->common.cookie; 8242dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 825b4dae6e1adaedc9c343b5f00332312d649600bdcGuennadi Liakhovetski spin_unlock_irqrestore(&sh_chan->desc_lock, flags); 8263542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 8273542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (callback) 8283542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski callback(param); 8293542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 8303542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski return callback; 8313542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski} 8323542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 8333542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski/* 8343542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * sh_chan_ld_cleanup - Clean up link descriptors 8353542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * 8363542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * This function cleans up the ld_queue of DMA channel. 8373542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski */ 8383542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetskistatic void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan, bool all) 8393542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski{ 8403542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski while (__ld_cleanup(sh_chan, all)) 8413542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski ; 842d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 843d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 8447a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski/* Called under spin_lock_irq(&sh_chan->desc_lock) */ 845d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan) 846d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 84747a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski struct sh_desc *desc; 848d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 849d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* DMA work check */ 8507a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski if (dmae_is_busy(sh_chan)) 851b4dae6e1adaedc9c343b5f00332312d649600bdcGuennadi Liakhovetski return; 852d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 8535a3a7658863f74f28cef53b9336bff7423659801Justin P. Mattock /* Find the first not transferred descriptor */ 85447a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski list_for_each_entry(desc, &sh_chan->ld_queue, node) 85547a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski if (desc->mark == DESC_SUBMITTED) { 856c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski dev_dbg(sh_chan->dev, "Queue #%d to %d: %u@%x -> %x\n", 857c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski desc->async_tx.cookie, sh_chan->id, 858c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski desc->hw.tcr, desc->hw.sar, desc->hw.dar); 8593542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Get the ld start address from ld_queue */ 86047a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski dmae_set_reg(sh_chan, &desc->hw); 8613542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dmae_start(sh_chan); 8623542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski break; 8633542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 864d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 865d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 866d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void sh_dmae_memcpy_issue_pending(struct dma_chan *chan) 867d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 868d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_chan *sh_chan = to_sh_chan(chan); 8697a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski 8707a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski spin_lock_irq(&sh_chan->desc_lock); 8717a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski if (sh_chan->pm_state == DMAE_PM_ESTABLISHED) 8727a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski sh_chan_xfer_ld_queue(sh_chan); 8737a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski else 8747a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski sh_chan->pm_state = DMAE_PM_PENDING; 8757a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski spin_unlock_irq(&sh_chan->desc_lock); 876d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 877d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 8780793448187643b50af89d36b08470baf45a3cab4Linus Walleijstatic enum dma_status sh_dmae_tx_status(struct dma_chan *chan, 879d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_cookie_t cookie, 8800793448187643b50af89d36b08470baf45a3cab4Linus Walleij struct dma_tx_state *txstate) 881d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 882d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_chan *sh_chan = to_sh_chan(chan); 88347a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski enum dma_status status; 884b4dae6e1adaedc9c343b5f00332312d649600bdcGuennadi Liakhovetski unsigned long flags; 885d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 8863542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski sh_dmae_chan_ld_cleanup(sh_chan, false); 887d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 888b4dae6e1adaedc9c343b5f00332312d649600bdcGuennadi Liakhovetski spin_lock_irqsave(&sh_chan->desc_lock, flags); 88947a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski 89096a2af41c78b1fbb1f567a3486bdc63f7b31c5fdRussell King - ARM Linux status = dma_cookie_status(chan, cookie, txstate); 89147a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski 89247a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski /* 89347a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski * If we don't find cookie on the queue, it has been aborted and we have 89447a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski * to report error 89547a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski */ 89647a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski if (status != DMA_SUCCESS) { 89747a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski struct sh_desc *desc; 89847a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski status = DMA_ERROR; 89947a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski list_for_each_entry(desc, &sh_chan->ld_queue, node) 90047a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski if (desc->cookie == cookie) { 90147a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski status = DMA_IN_PROGRESS; 90247a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski break; 90347a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski } 90447a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski } 90547a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski 906b4dae6e1adaedc9c343b5f00332312d649600bdcGuennadi Liakhovetski spin_unlock_irqrestore(&sh_chan->desc_lock, flags); 90747a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski 90847a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski return status; 909d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 910d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 911d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic irqreturn_t sh_dmae_interrupt(int irq, void *data) 912d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 913d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu irqreturn_t ret = IRQ_NONE; 9142dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski struct sh_dmae_chan *sh_chan = data; 9152dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski u32 chcr; 9162dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 9172dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_lock(&sh_chan->desc_lock); 9182dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 9195899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto chcr = chcr_read(sh_chan); 920d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 921d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (chcr & CHCR_TE) { 922d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* DMA stop */ 923d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dmae_halt(sh_chan); 924d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 925d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu ret = IRQ_HANDLED; 926d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu tasklet_schedule(&sh_chan->tasklet); 927d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 928d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 9292dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_unlock(&sh_chan->desc_lock); 9302dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 931d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return ret; 932d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 933d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 9342dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski/* Called from error IRQ or NMI */ 9352dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetskistatic bool sh_dmae_reset(struct sh_dmae_device *shdev) 936d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 93703aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt unsigned int handled = 0; 93847a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski int i; 939d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 94047a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski /* halt the dma controller */ 941027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski sh_dmae_ctl_stop(shdev); 94247a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski 94347a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski /* We cannot detect, which channel caused the error, have to reset all */ 9448b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski for (i = 0; i < SH_DMAC_MAX_CHANNELS; i++) { 94547a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski struct sh_dmae_chan *sh_chan = shdev->chan[i]; 94603aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt struct sh_desc *desc; 9472dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski LIST_HEAD(dl); 94803aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 94903aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt if (!sh_chan) 95003aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt continue; 95103aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 9522dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_lock(&sh_chan->desc_lock); 9532dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 95403aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt /* Stop the channel */ 95503aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt dmae_halt(sh_chan); 95603aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 9572dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski list_splice_init(&sh_chan->ld_queue, &dl); 9582dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 9597a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski if (!list_empty(&dl)) { 9607a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski dev_dbg(sh_chan->dev, "Bring down channel %d\n", sh_chan->id); 9617a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski pm_runtime_put(sh_chan->dev); 9627a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski } 9637a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski sh_chan->pm_state = DMAE_PM_ESTABLISHED; 9647a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski 9652dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_unlock(&sh_chan->desc_lock); 9662dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 96703aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt /* Complete all */ 9682dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski list_for_each_entry(desc, &dl, node) { 96903aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt struct dma_async_tx_descriptor *tx = &desc->async_tx; 97003aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt desc->mark = DESC_IDLE; 97103aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt if (tx->callback) 97203aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt tx->callback(tx->callback_param); 973d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 97403aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 9752dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_lock(&sh_chan->desc_lock); 9762dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski list_splice(&dl, &sh_chan->ld_free); 9772dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_unlock(&sh_chan->desc_lock); 9782dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 97903aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt handled++; 980d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 98103aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 982027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski sh_dmae_rst(shdev); 98347a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski 98403aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt return !!handled; 98503aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt} 98603aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 98703aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundtstatic irqreturn_t sh_dmae_err(int irq, void *data) 98803aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt{ 989ff7690b48ae8571d930a2621e21f6e5a41e42b6dYoshihiro Shimoda struct sh_dmae_device *shdev = data; 990ff7690b48ae8571d930a2621e21f6e5a41e42b6dYoshihiro Shimoda 9912dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski if (!(dmaor_read(shdev) & DMAOR_AE)) 992ff7690b48ae8571d930a2621e21f6e5a41e42b6dYoshihiro Shimoda return IRQ_NONE; 9932dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 9942dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski sh_dmae_reset(data); 9952dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski return IRQ_HANDLED; 996d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 997d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 998d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void dmae_do_tasklet(unsigned long data) 999d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1000d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_chan *sh_chan = (struct sh_dmae_chan *)data; 10013542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski struct sh_desc *desc; 1002d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu u32 sar_buf = sh_dmae_readl(sh_chan, SAR); 1003cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski u32 dar_buf = sh_dmae_readl(sh_chan, DAR); 100486d61b33e48f1da5a6b310d3de93187db62ab72aGuennadi Liakhovetski 1005b4dae6e1adaedc9c343b5f00332312d649600bdcGuennadi Liakhovetski spin_lock_irq(&sh_chan->desc_lock); 10063542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski list_for_each_entry(desc, &sh_chan->ld_queue, node) { 1007cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski if (desc->mark == DESC_SUBMITTED && 1008db8196df4bb6f117caa163aa73b0f16fd62290bdVinod Koul ((desc->direction == DMA_DEV_TO_MEM && 1009cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski (desc->hw.dar + desc->hw.tcr) == dar_buf) || 1010cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski (desc->hw.sar + desc->hw.tcr) == sar_buf)) { 10113542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dev_dbg(sh_chan->dev, "done #%d@%p dst %u\n", 10123542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->async_tx.cookie, &desc->async_tx, 10133542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->hw.dar); 10143542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark = DESC_COMPLETED; 1015d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu break; 1016d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1017d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1018d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* Next desc */ 1019d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu sh_chan_xfer_ld_queue(sh_chan); 10207a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski spin_unlock_irq(&sh_chan->desc_lock); 10217a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski 10223542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski sh_dmae_chan_ld_cleanup(sh_chan, false); 1023d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 1024d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 102503aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundtstatic bool sh_dmae_nmi_notify(struct sh_dmae_device *shdev) 102603aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt{ 102703aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt /* Fast path out if NMIF is not asserted for this controller */ 102803aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt if ((dmaor_read(shdev) & DMAOR_NMIF) == 0) 102903aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt return false; 103003aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 10312dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski return sh_dmae_reset(shdev); 103203aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt} 103303aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 103403aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundtstatic int sh_dmae_nmi_handler(struct notifier_block *self, 103503aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt unsigned long cmd, void *data) 103603aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt{ 103703aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt struct sh_dmae_device *shdev; 103803aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt int ret = NOTIFY_DONE; 103903aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt bool triggered; 104003aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 104103aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt /* 104203aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt * Only concern ourselves with NMI events. 104303aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt * 104403aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt * Normally we would check the die chain value, but as this needs 104503aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt * to be architecture independent, check for NMI context instead. 104603aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt */ 104703aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt if (!in_nmi()) 104803aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt return NOTIFY_DONE; 104903aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 105003aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt rcu_read_lock(); 105103aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt list_for_each_entry_rcu(shdev, &sh_dmae_devices, node) { 105203aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt /* 105303aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt * Only stop if one of the controllers has NMIF asserted, 105403aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt * we do not want to interfere with regular address error 105503aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt * handling or NMI events that don't concern the DMACs. 105603aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt */ 105703aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt triggered = sh_dmae_nmi_notify(shdev); 105803aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt if (triggered == true) 105903aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt ret = NOTIFY_OK; 106003aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt } 106103aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt rcu_read_unlock(); 106203aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 106303aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt return ret; 106403aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt} 106503aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 106603aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundtstatic struct notifier_block sh_dmae_nmi_notifier __read_mostly = { 106703aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt .notifier_call = sh_dmae_nmi_handler, 106803aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 106903aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt /* Run before NMI debug handler and KGDB */ 107003aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt .priority = 1, 107103aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt}; 107203aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 1073027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskistatic int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id, 1074027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski int irq, unsigned long flags) 1075d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1076d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu int err; 10775bac942db3d2c4738df04104240d65a5d1eaec6aGuennadi Liakhovetski const struct sh_dmae_channel *chan_pdata = &shdev->pdata->channel[id]; 1078027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct platform_device *pdev = to_platform_device(shdev->common.dev); 1079d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_chan *new_sh_chan; 1080d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1081d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* alloc channel */ 1082d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu new_sh_chan = kzalloc(sizeof(struct sh_dmae_chan), GFP_KERNEL); 1083d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (!new_sh_chan) { 108486d61b33e48f1da5a6b310d3de93187db62ab72aGuennadi Liakhovetski dev_err(shdev->common.dev, 108586d61b33e48f1da5a6b310d3de93187db62ab72aGuennadi Liakhovetski "No free memory for allocating dma channels!\n"); 1086d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return -ENOMEM; 1087d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1088d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 10897a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski new_sh_chan->pm_state = DMAE_PM_ESTABLISHED; 10907a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski 10917a1cd9ad87979744e1510782b25c38feb9602739Guennadi Liakhovetski /* reference struct dma_device */ 10928b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski new_sh_chan->common.device = &shdev->common; 10938ac695463f37af902e953d575d3f782e32e170daRussell King - ARM Linux dma_cookie_init(&new_sh_chan->common); 10948b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski 1095d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu new_sh_chan->dev = shdev->common.dev; 1096d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu new_sh_chan->id = id; 1097027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski new_sh_chan->irq = irq; 1098027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski new_sh_chan->base = shdev->chan_reg + chan_pdata->offset / sizeof(u32); 1099d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1100d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* Init DMA tasklet */ 1101d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu tasklet_init(&new_sh_chan->tasklet, dmae_do_tasklet, 1102d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu (unsigned long)new_sh_chan); 1103d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1104d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_lock_init(&new_sh_chan->desc_lock); 1105d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1106d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* Init descripter manage list */ 1107d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu INIT_LIST_HEAD(&new_sh_chan->ld_queue); 1108d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu INIT_LIST_HEAD(&new_sh_chan->ld_free); 1109d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1110d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* Add the channel to DMA device channel list */ 1111d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu list_add_tail(&new_sh_chan->common.device_node, 1112d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu &shdev->common.channels); 1113d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->common.chancnt++; 1114d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1115027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (pdev->id >= 0) 1116027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski snprintf(new_sh_chan->dev_id, sizeof(new_sh_chan->dev_id), 1117027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski "sh-dmae%d.%d", pdev->id, new_sh_chan->id); 1118027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski else 1119027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski snprintf(new_sh_chan->dev_id, sizeof(new_sh_chan->dev_id), 1120027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski "sh-dma%d", new_sh_chan->id); 1121d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1122d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* set up channel irq */ 1123027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski err = request_irq(irq, &sh_dmae_interrupt, flags, 112486d61b33e48f1da5a6b310d3de93187db62ab72aGuennadi Liakhovetski new_sh_chan->dev_id, new_sh_chan); 1125d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (err) { 1126d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dev_err(shdev->common.dev, "DMA channel %d request_irq error " 1127d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu "with return %d\n", id, err); 1128d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu goto err_no_irq; 1129d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1130d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1131d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->chan[id] = new_sh_chan; 1132d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return 0; 1133d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1134d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsuerr_no_irq: 1135d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* remove from dmaengine device node */ 1136d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu list_del(&new_sh_chan->common.device_node); 1137d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu kfree(new_sh_chan); 1138d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return err; 1139d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 1140d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1141d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void sh_dmae_chan_remove(struct sh_dmae_device *shdev) 1142d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1143d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu int i; 1144d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1145d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu for (i = shdev->common.chancnt - 1 ; i >= 0 ; i--) { 1146d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (shdev->chan[i]) { 1147027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct sh_dmae_chan *sh_chan = shdev->chan[i]; 1148027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1149027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski free_irq(sh_chan->irq, sh_chan); 1150d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1151027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski list_del(&sh_chan->common.device_node); 1152027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski kfree(sh_chan); 1153d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->chan[i] = NULL; 1154d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1155d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1156d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->common.chancnt = 0; 1157d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 1158d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1159d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic int __init sh_dmae_probe(struct platform_device *pdev) 1160d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1161027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct sh_dmae_pdata *pdata = pdev->dev.platform_data; 1162027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski unsigned long irqflags = IRQF_DISABLED, 11638b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski chan_flag[SH_DMAC_MAX_CHANNELS] = {}; 11648b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski int errirq, chan_irq[SH_DMAC_MAX_CHANNELS]; 1165300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm int err, i, irq_cnt = 0, irqres = 0, irq_cap = 0; 1166d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_device *shdev; 1167027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct resource *chan, *dmars, *errirq_res, *chanirq_res; 1168d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 116956adf7e8127d601b172e180b44551ce83404348fDan Williams /* get platform data */ 1170027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (!pdata || !pdata->channel_num) 117156adf7e8127d601b172e180b44551ce83404348fDan Williams return -ENODEV; 117256adf7e8127d601b172e180b44551ce83404348fDan Williams 1173027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chan = platform_get_resource(pdev, IORESOURCE_MEM, 0); 117426fc02ab5551349b2e593829a76cb44328ee7f61Magnus Damm /* DMARS area is optional */ 1175027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dmars = platform_get_resource(pdev, IORESOURCE_MEM, 1); 1176027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski /* 1177027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * IRQ resources: 1178027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * 1. there always must be at least one IRQ IO-resource. On SH4 it is 1179027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * the error IRQ, in which case it is the only IRQ in this resource: 1180027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * start == end. If it is the only IRQ resource, all channels also 1181027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * use the same IRQ. 1182027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * 2. DMA channel IRQ resources can be specified one per resource or in 1183027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * ranges (start != end) 1184027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * 3. iff all events (channels and, optionally, error) on this 1185027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * controller use the same IRQ, only one IRQ resource can be 1186027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * specified, otherwise there must be one IRQ per channel, even if 1187027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * some of them are equal 1188027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * 4. if all IRQs on this controller are equal or if some specific IRQs 1189027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * specify IORESOURCE_IRQ_SHAREABLE in their resources, they will be 1190027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * requested with the IRQF_SHARED flag 1191027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski */ 1192027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski errirq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); 1193027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (!chan || !errirq_res) 1194027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski return -ENODEV; 1195027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1196027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (!request_mem_region(chan->start, resource_size(chan), pdev->name)) { 1197027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dev_err(&pdev->dev, "DMAC register region already claimed\n"); 1198027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski return -EBUSY; 1199027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski } 1200027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1201027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (dmars && !request_mem_region(dmars->start, resource_size(dmars), pdev->name)) { 1202027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dev_err(&pdev->dev, "DMAC DMARS region already claimed\n"); 1203027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski err = -EBUSY; 1204027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski goto ermrdmars; 1205027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski } 1206027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1207027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski err = -ENOMEM; 1208d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev = kzalloc(sizeof(struct sh_dmae_device), GFP_KERNEL); 1209d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (!shdev) { 1210027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dev_err(&pdev->dev, "Not enough memory\n"); 1211027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski goto ealloc; 1212027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski } 1213027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1214027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski shdev->chan_reg = ioremap(chan->start, resource_size(chan)); 1215027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (!shdev->chan_reg) 1216027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski goto emapchan; 1217027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (dmars) { 1218027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski shdev->dmars = ioremap(dmars->start, resource_size(dmars)); 1219027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (!shdev->dmars) 1220027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski goto emapdmars; 1221d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1222d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1223d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* platform data */ 1224027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski shdev->pdata = pdata; 1225d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 12265899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto if (pdata->chcr_offset) 12275899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto shdev->chcr_offset = pdata->chcr_offset; 12285899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto else 12295899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto shdev->chcr_offset = CHCR; 12305899a723b336da241b492583d7e55f1055f8f3b3Kuninori Morimoto 123167c6269e5c998b53c2c08ce2befbbe20a7b6f57fKuninori Morimoto if (pdata->chcr_ie_bit) 123267c6269e5c998b53c2c08ce2befbbe20a7b6f57fKuninori Morimoto shdev->chcr_ie_bit = pdata->chcr_ie_bit; 123367c6269e5c998b53c2c08ce2befbbe20a7b6f57fKuninori Morimoto else 123467c6269e5c998b53c2c08ce2befbbe20a7b6f57fKuninori Morimoto shdev->chcr_ie_bit = CHCR_IE; 123567c6269e5c998b53c2c08ce2befbbe20a7b6f57fKuninori Morimoto 12365c2de44417523385010b529599a2b30f290831a3Paul Mundt platform_set_drvdata(pdev, shdev); 12375c2de44417523385010b529599a2b30f290831a3Paul Mundt 1238c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski shdev->common.dev = &pdev->dev; 1239c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski 124020f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski pm_runtime_enable(&pdev->dev); 124120f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski pm_runtime_get_sync(&pdev->dev); 124220f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski 124331705e21f9b5a0628c043f88ff4d20488b47b8abGuennadi Liakhovetski spin_lock_irq(&sh_dmae_lock); 124403aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt list_add_tail_rcu(&shdev->node, &sh_dmae_devices); 124531705e21f9b5a0628c043f88ff4d20488b47b8abGuennadi Liakhovetski spin_unlock_irq(&sh_dmae_lock); 124603aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 12472dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski /* reset dma controller - only needed as a test */ 1248027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski err = sh_dmae_rst(shdev); 1249d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (err) 1250d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu goto rst_err; 1251d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1252d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu INIT_LIST_HEAD(&shdev->common.channels); 1253d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1254e9c8d7a03e69093e4c33c5056a45c1233a42e8a4Guennadi Liakhovetski if (!pdata->slave_only) 1255e9c8d7a03e69093e4c33c5056a45c1233a42e8a4Guennadi Liakhovetski dma_cap_set(DMA_MEMCPY, shdev->common.cap_mask); 125626fc02ab5551349b2e593829a76cb44328ee7f61Magnus Damm if (pdata->slave && pdata->slave_num) 1257027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dma_cap_set(DMA_SLAVE, shdev->common.cap_mask); 1258cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 1259d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->common.device_alloc_chan_resources 1260d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu = sh_dmae_alloc_chan_resources; 1261d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->common.device_free_chan_resources = sh_dmae_free_chan_resources; 1262d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->common.device_prep_dma_memcpy = sh_dmae_prep_memcpy; 12630793448187643b50af89d36b08470baf45a3cab4Linus Walleij shdev->common.device_tx_status = sh_dmae_tx_status; 1264d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->common.device_issue_pending = sh_dmae_memcpy_issue_pending; 1265cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 1266cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski /* Compulsory for DMA_SLAVE fields */ 1267cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski shdev->common.device_prep_slave_sg = sh_dmae_prep_slave_sg; 1268c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij shdev->common.device_control = sh_dmae_control; 1269cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 1270ddb4f0f0e05871c7ac540cc778993c06ff53b765Guennadi Liakhovetski /* Default transfer size of 32 bytes requires 32-byte alignment */ 12718b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski shdev->common.copy_align = LOG2_DEFAULT_XFER_SIZE; 1272d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1273927a7c9c1793def3a55d60c926d3945528e6bf1bMagnus Damm#if defined(CONFIG_CPU_SH4) || defined(CONFIG_ARCH_SHMOBILE) 1274027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chanirq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 1); 1275027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1276027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (!chanirq_res) 1277027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chanirq_res = errirq_res; 1278027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski else 1279027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski irqres++; 1280027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1281027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (chanirq_res == errirq_res || 1282027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski (errirq_res->flags & IORESOURCE_BITS) == IORESOURCE_IRQ_SHAREABLE) 1283d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu irqflags = IRQF_SHARED; 1284027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1285027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski errirq = errirq_res->start; 1286027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1287027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski err = request_irq(errirq, sh_dmae_err, irqflags, 1288027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski "DMAC Address Error", shdev); 1289027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (err) { 1290027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dev_err(&pdev->dev, 1291027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski "DMA failed requesting irq #%d, error %d\n", 1292027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski errirq, err); 1293027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski goto eirq_err; 1294d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1295d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1296027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski#else 1297027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chanirq_res = errirq_res; 1298927a7c9c1793def3a55d60c926d3945528e6bf1bMagnus Damm#endif /* CONFIG_CPU_SH4 || CONFIG_ARCH_SHMOBILE */ 1299027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1300027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (chanirq_res->start == chanirq_res->end && 1301027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski !platform_get_resource(pdev, IORESOURCE_IRQ, 1)) { 1302027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski /* Special case - all multiplexed */ 1303027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski for (; irq_cnt < pdata->channel_num; irq_cnt++) { 1304300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm if (irq_cnt < SH_DMAC_MAX_CHANNELS) { 1305300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm chan_irq[irq_cnt] = chanirq_res->start; 1306300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm chan_flag[irq_cnt] = IRQF_SHARED; 1307300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm } else { 1308300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm irq_cap = 1; 1309300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm break; 1310300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm } 1311d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1312027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski } else { 1313027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski do { 1314027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski for (i = chanirq_res->start; i <= chanirq_res->end; i++) { 1315dcee0bb713d0ba0d32c5ce6fe0c5aa22e6fc274aMagnus Damm if (irq_cnt >= SH_DMAC_MAX_CHANNELS) { 1316dcee0bb713d0ba0d32c5ce6fe0c5aa22e6fc274aMagnus Damm irq_cap = 1; 1317dcee0bb713d0ba0d32c5ce6fe0c5aa22e6fc274aMagnus Damm break; 1318dcee0bb713d0ba0d32c5ce6fe0c5aa22e6fc274aMagnus Damm } 1319dcee0bb713d0ba0d32c5ce6fe0c5aa22e6fc274aMagnus Damm 1320027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if ((errirq_res->flags & IORESOURCE_BITS) == 1321027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski IORESOURCE_IRQ_SHAREABLE) 1322027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chan_flag[irq_cnt] = IRQF_SHARED; 1323027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski else 1324027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chan_flag[irq_cnt] = IRQF_DISABLED; 1325027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dev_dbg(&pdev->dev, 1326027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski "Found IRQ %d for channel %d\n", 1327027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski i, irq_cnt); 1328027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chan_irq[irq_cnt++] = i; 1329300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm } 1330300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm 1331dcee0bb713d0ba0d32c5ce6fe0c5aa22e6fc274aMagnus Damm if (irq_cnt >= SH_DMAC_MAX_CHANNELS) 1332300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm break; 1333dcee0bb713d0ba0d32c5ce6fe0c5aa22e6fc274aMagnus Damm 1334027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chanirq_res = platform_get_resource(pdev, 1335027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski IORESOURCE_IRQ, ++irqres); 1336027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski } while (irq_cnt < pdata->channel_num && chanirq_res); 1337d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1338027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1339d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* Create DMA Channel */ 1340300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm for (i = 0; i < irq_cnt; i++) { 1341027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski err = sh_dmae_chan_probe(shdev, i, chan_irq[i], chan_flag[i]); 1342d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (err) 1343d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu goto chan_probe_err; 1344d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1345d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1346300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm if (irq_cap) 1347300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm dev_notice(&pdev->dev, "Attempting to register %d DMA " 1348300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm "channels when a maximum of %d are supported.\n", 1349300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm pdata->channel_num, SH_DMAC_MAX_CHANNELS); 1350300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm 135120f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski pm_runtime_put(&pdev->dev); 135220f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski 1353d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_async_device_register(&shdev->common); 1354d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1355d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return err; 1356d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1357d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsuchan_probe_err: 1358d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu sh_dmae_chan_remove(shdev); 1359300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm 1360927a7c9c1793def3a55d60c926d3945528e6bf1bMagnus Damm#if defined(CONFIG_CPU_SH4) || defined(CONFIG_ARCH_SHMOBILE) 1361027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski free_irq(errirq, shdev); 1362d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsueirq_err: 1363027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski#endif 1364d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsurst_err: 136531705e21f9b5a0628c043f88ff4d20488b47b8abGuennadi Liakhovetski spin_lock_irq(&sh_dmae_lock); 136603aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt list_del_rcu(&shdev->node); 136731705e21f9b5a0628c043f88ff4d20488b47b8abGuennadi Liakhovetski spin_unlock_irq(&sh_dmae_lock); 136803aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 136920f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski pm_runtime_put(&pdev->dev); 1370467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski pm_runtime_disable(&pdev->dev); 1371467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1372027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (dmars) 1373027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski iounmap(shdev->dmars); 13745c2de44417523385010b529599a2b30f290831a3Paul Mundt 13755c2de44417523385010b529599a2b30f290831a3Paul Mundt platform_set_drvdata(pdev, NULL); 1376027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskiemapdmars: 1377027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski iounmap(shdev->chan_reg); 137831705e21f9b5a0628c043f88ff4d20488b47b8abGuennadi Liakhovetski synchronize_rcu(); 1379027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskiemapchan: 1380d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu kfree(shdev); 1381027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskiealloc: 1382027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (dmars) 1383027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski release_mem_region(dmars->start, resource_size(dmars)); 1384027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskiermrdmars: 1385027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski release_mem_region(chan->start, resource_size(chan)); 1386d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1387d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return err; 1388d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 1389d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1390d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic int __exit sh_dmae_remove(struct platform_device *pdev) 1391d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1392d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_device *shdev = platform_get_drvdata(pdev); 1393027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct resource *res; 1394027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski int errirq = platform_get_irq(pdev, 0); 1395d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1396d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_async_device_unregister(&shdev->common); 1397d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1398027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (errirq > 0) 1399027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski free_irq(errirq, shdev); 1400d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 140131705e21f9b5a0628c043f88ff4d20488b47b8abGuennadi Liakhovetski spin_lock_irq(&sh_dmae_lock); 140203aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt list_del_rcu(&shdev->node); 140331705e21f9b5a0628c043f88ff4d20488b47b8abGuennadi Liakhovetski spin_unlock_irq(&sh_dmae_lock); 140403aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 1405d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* channel data remove */ 1406d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu sh_dmae_chan_remove(shdev); 1407d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 140820f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski pm_runtime_disable(&pdev->dev); 140920f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski 1410027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (shdev->dmars) 1411027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski iounmap(shdev->dmars); 1412027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski iounmap(shdev->chan_reg); 1413027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 14145c2de44417523385010b529599a2b30f290831a3Paul Mundt platform_set_drvdata(pdev, NULL); 14155c2de44417523385010b529599a2b30f290831a3Paul Mundt 141631705e21f9b5a0628c043f88ff4d20488b47b8abGuennadi Liakhovetski synchronize_rcu(); 1417d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu kfree(shdev); 1418d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1419027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1420027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (res) 1421027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski release_mem_region(res->start, resource_size(res)); 1422027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski res = platform_get_resource(pdev, IORESOURCE_MEM, 1); 1423027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (res) 1424027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski release_mem_region(res->start, resource_size(res)); 1425027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1426d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return 0; 1427d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 1428d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1429d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void sh_dmae_shutdown(struct platform_device *pdev) 1430d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1431d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_device *shdev = platform_get_drvdata(pdev); 1432027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski sh_dmae_ctl_stop(shdev); 1433d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 1434d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1435467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetskistatic int sh_dmae_runtime_suspend(struct device *dev) 1436467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski{ 1437467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski return 0; 1438467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski} 1439467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1440467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetskistatic int sh_dmae_runtime_resume(struct device *dev) 1441467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski{ 1442467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski struct sh_dmae_device *shdev = dev_get_drvdata(dev); 1443467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1444467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski return sh_dmae_rst(shdev); 1445467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski} 1446467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1447467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski#ifdef CONFIG_PM 1448467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetskistatic int sh_dmae_suspend(struct device *dev) 1449467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski{ 1450467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski return 0; 1451467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski} 1452467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1453467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetskistatic int sh_dmae_resume(struct device *dev) 1454467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski{ 1455467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski struct sh_dmae_device *shdev = dev_get_drvdata(dev); 1456c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski int i, ret; 1457c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski 1458c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski ret = sh_dmae_rst(shdev); 1459c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski if (ret < 0) 1460c11b46c32c8a9bf05fdb76d70d8dc74fcbfd02d1Guennadi Liakhovetski dev_err(dev, "Failed to reset!\n"); 1461467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1462467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski for (i = 0; i < shdev->pdata->channel_num; i++) { 1463467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski struct sh_dmae_chan *sh_chan = shdev->chan[i]; 1464467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski struct sh_dmae_slave *param = sh_chan->common.private; 1465467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1466467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski if (!sh_chan->descs_allocated) 1467467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski continue; 1468467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1469467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski if (param) { 1470467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski const struct sh_dmae_slave_config *cfg = param->config; 1471467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski dmae_set_dmars(sh_chan, cfg->mid_rid); 1472467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski dmae_set_chcr(sh_chan, cfg->chcr); 1473467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski } else { 1474467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski dmae_init(sh_chan); 1475467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski } 1476467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski } 1477467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1478467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski return 0; 1479467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski} 1480467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski#else 1481467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski#define sh_dmae_suspend NULL 1482467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski#define sh_dmae_resume NULL 1483467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski#endif 1484467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1485467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetskiconst struct dev_pm_ops sh_dmae_pm = { 1486467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski .suspend = sh_dmae_suspend, 1487467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski .resume = sh_dmae_resume, 1488467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski .runtime_suspend = sh_dmae_runtime_suspend, 1489467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski .runtime_resume = sh_dmae_runtime_resume, 1490467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski}; 1491467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1492d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic struct platform_driver sh_dmae_driver = { 1493d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu .remove = __exit_p(sh_dmae_remove), 1494d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu .shutdown = sh_dmae_shutdown, 1495d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu .driver = { 14967a5c106a0e8fd03a806d0da77eef10b4045c43a6Guennadi Liakhovetski .owner = THIS_MODULE, 1497d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu .name = "sh-dma-engine", 1498467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski .pm = &sh_dmae_pm, 1499d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu }, 1500d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu}; 1501d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1502d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic int __init sh_dmae_init(void) 1503d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1504661382fe190475c17d0b3a6b5f0350b4f82f5939Guennadi Liakhovetski /* Wire up NMI handling */ 1505661382fe190475c17d0b3a6b5f0350b4f82f5939Guennadi Liakhovetski int err = register_die_notifier(&sh_dmae_nmi_notifier); 1506661382fe190475c17d0b3a6b5f0350b4f82f5939Guennadi Liakhovetski if (err) 1507661382fe190475c17d0b3a6b5f0350b4f82f5939Guennadi Liakhovetski return err; 1508661382fe190475c17d0b3a6b5f0350b4f82f5939Guennadi Liakhovetski 1509d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return platform_driver_probe(&sh_dmae_driver, sh_dmae_probe); 1510d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 1511d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsumodule_init(sh_dmae_init); 1512d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1513d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void __exit sh_dmae_exit(void) 1514d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1515d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu platform_driver_unregister(&sh_dmae_driver); 1516661382fe190475c17d0b3a6b5f0350b4f82f5939Guennadi Liakhovetski 1517661382fe190475c17d0b3a6b5f0350b4f82f5939Guennadi Liakhovetski unregister_die_notifier(&sh_dmae_nmi_notifier); 1518d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 1519d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsumodule_exit(sh_dmae_exit); 1520d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1521d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro IwamatsuMODULE_AUTHOR("Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com>"); 1522d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro IwamatsuMODULE_DESCRIPTION("Renesas SH DMA Engine driver"); 1523d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro IwamatsuMODULE_LICENSE("GPL"); 1524e5843341e3ad8ff00332376cd0745026e4b5d45fGuennadi LiakhovetskiMODULE_ALIAS("platform:sh-dma-engine"); 1525