shdma.c revision 4bab9d426e6dbd9ea09330919a33d35d5faab400
1d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu/* 2d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * Renesas SuperH DMA Engine support 3d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * 4d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * base is drivers/dma/flsdma.c 5d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * 6d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * Copyright (C) 2009 Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com> 7d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * Copyright (C) 2009 Renesas Solutions, Inc. All rights reserved. 8d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved. 9d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * 10d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * This is free software; you can redistribute it and/or modify 11d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * it under the terms of the GNU General Public License as published by 12d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * the Free Software Foundation; either version 2 of the License, or 13d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * (at your option) any later version. 14d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * 15d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * - DMA of SuperH does not have Hardware DMA chain mode. 16d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * - MAX DMA size is 16MB. 17d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * 18d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu */ 19d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 20d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include <linux/init.h> 21d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include <linux/module.h> 22d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include <linux/interrupt.h> 23d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include <linux/dmaengine.h> 24d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include <linux/delay.h> 25d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include <linux/dma-mapping.h> 26d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include <linux/platform_device.h> 2720f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski#include <linux/pm_runtime.h> 2820f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski 298b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski#include <asm/dmaengine.h> 3020f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski 31d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include "shdma.h" 32d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 33d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu/* DMA descriptor control */ 343542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetskienum sh_dmae_desc_status { 353542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski DESC_IDLE, 363542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski DESC_PREPARED, 373542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski DESC_SUBMITTED, 383542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski DESC_COMPLETED, /* completed, have to call callback */ 393542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski DESC_WAITING, /* callback called, waiting for ack / re-submit */ 403542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski}; 41d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 42d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#define NR_DESCS_PER_CHANNEL 32 438b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski/* Default MEMCPY transfer size = 2^2 = 4 bytes */ 448b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski#define LOG2_DEFAULT_XFER_SIZE 2 45d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 46cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski/* A bitmask with bits enough for enum sh_dmae_slave_chan_id */ 47cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetskistatic unsigned long sh_dmae_slave_used[BITS_TO_LONGS(SHDMA_SLAVE_NUMBER)]; 48cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 493542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetskistatic void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan, bool all); 503542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 51d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void sh_dmae_writel(struct sh_dmae_chan *sh_dc, u32 data, u32 reg) 52d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 53027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski __raw_writel(data, sh_dc->base + reg / sizeof(u32)); 54d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 55d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 56d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic u32 sh_dmae_readl(struct sh_dmae_chan *sh_dc, u32 reg) 57d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 58027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski return __raw_readl(sh_dc->base + reg / sizeof(u32)); 59027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski} 60027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 61027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskistatic u16 dmaor_read(struct sh_dmae_device *shdev) 62027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski{ 63027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski return __raw_readw(shdev->chan_reg + DMAOR / sizeof(u32)); 64027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski} 65027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 66027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskistatic void dmaor_write(struct sh_dmae_device *shdev, u16 data) 67027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski{ 68027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski __raw_writew(data, shdev->chan_reg + DMAOR / sizeof(u32)); 69d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 70d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 71d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu/* 72d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * Reset DMA controller 73d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * 74d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * SH7780 has two DMAOR register 75d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu */ 76027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskistatic void sh_dmae_ctl_stop(struct sh_dmae_device *shdev) 77d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 78027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski unsigned short dmaor = dmaor_read(shdev); 79d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 80027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dmaor_write(shdev, dmaor & ~(DMAOR_NMIF | DMAOR_AE | DMAOR_DME)); 81d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 82d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 83027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskistatic int sh_dmae_rst(struct sh_dmae_device *shdev) 84d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 85d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu unsigned short dmaor; 86d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 87027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski sh_dmae_ctl_stop(shdev); 888b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski dmaor = dmaor_read(shdev) | shdev->pdata->dmaor_init; 89d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 90027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dmaor_write(shdev, dmaor); 91027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (dmaor_read(shdev) & (DMAOR_AE | DMAOR_NMIF)) { 9247a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski pr_warning("dma-sh: Can't initialize DMAOR.\n"); 93d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return -EINVAL; 94d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 95d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return 0; 96d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 97d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 98fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetskistatic bool dmae_is_busy(struct sh_dmae_chan *sh_chan) 99d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 100d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu u32 chcr = sh_dmae_readl(sh_chan, CHCR); 101fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 102fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if ((chcr & (CHCR_DE | CHCR_TE)) == CHCR_DE) 103fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return true; /* working */ 104fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 105fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return false; /* waiting */ 106d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 107d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1088b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetskistatic unsigned int calc_xmit_shift(struct sh_dmae_chan *sh_chan, u32 chcr) 109d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1108b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski struct sh_dmae_device *shdev = container_of(sh_chan->common.device, 1118b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski struct sh_dmae_device, common); 1128b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski struct sh_dmae_pdata *pdata = shdev->pdata; 1138b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski int cnt = ((chcr & pdata->ts_low_mask) >> pdata->ts_low_shift) | 1148b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski ((chcr & pdata->ts_high_mask) >> pdata->ts_high_shift); 1158b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski 1168b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski if (cnt >= pdata->ts_shift_num) 1178b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski cnt = 0; 118623b4ac4bf9e767991c66e29b47dd4b19458fb42Guennadi Liakhovetski 1198b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski return pdata->ts_shift[cnt]; 1208b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski} 1218b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski 1228b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetskistatic u32 log2size_to_chcr(struct sh_dmae_chan *sh_chan, int l2size) 1238b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski{ 1248b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski struct sh_dmae_device *shdev = container_of(sh_chan->common.device, 1258b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski struct sh_dmae_device, common); 1268b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski struct sh_dmae_pdata *pdata = shdev->pdata; 1278b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski int i; 1288b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski 1298b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski for (i = 0; i < pdata->ts_shift_num; i++) 1308b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski if (pdata->ts_shift[i] == l2size) 1318b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski break; 1328b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski 1338b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski if (i == pdata->ts_shift_num) 1348b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski i = 0; 1358b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski 1368b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski return ((i << pdata->ts_low_shift) & pdata->ts_low_mask) | 1378b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski ((i << pdata->ts_high_shift) & pdata->ts_high_mask); 138d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 139d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1403542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetskistatic void dmae_set_reg(struct sh_dmae_chan *sh_chan, struct sh_dmae_regs *hw) 141d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1423542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski sh_dmae_writel(sh_chan, hw->sar, SAR); 1433542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski sh_dmae_writel(sh_chan, hw->dar, DAR); 144cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski sh_dmae_writel(sh_chan, hw->tcr >> sh_chan->xmit_shift, TCR); 145d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 146d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 147d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void dmae_start(struct sh_dmae_chan *sh_chan) 148d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 149d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu u32 chcr = sh_dmae_readl(sh_chan, CHCR); 150d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 15186d61b33e48f1da5a6b310d3de93187db62ab72aGuennadi Liakhovetski chcr |= CHCR_DE | CHCR_IE; 152cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski sh_dmae_writel(sh_chan, chcr & ~CHCR_TE, CHCR); 153d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 154d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 155d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void dmae_halt(struct sh_dmae_chan *sh_chan) 156d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 157d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu u32 chcr = sh_dmae_readl(sh_chan, CHCR); 158d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 159d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu chcr &= ~(CHCR_DE | CHCR_TE | CHCR_IE); 160d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu sh_dmae_writel(sh_chan, chcr, CHCR); 161d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 162d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 163cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetskistatic void dmae_init(struct sh_dmae_chan *sh_chan) 164cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski{ 1658b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski /* 1668b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski * Default configuration for dual address memory-memory transfer. 1678b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski * 0x400 represents auto-request. 1688b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski */ 1698b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski u32 chcr = DM_INC | SM_INC | 0x400 | log2size_to_chcr(sh_chan, 1708b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski LOG2_DEFAULT_XFER_SIZE); 1718b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski sh_chan->xmit_shift = calc_xmit_shift(sh_chan, chcr); 172cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski sh_dmae_writel(sh_chan, chcr, CHCR); 173cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski} 174cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 175d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic int dmae_set_chcr(struct sh_dmae_chan *sh_chan, u32 val) 176d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 177d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* When DMA was working, can not set data to CHCR */ 178fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (dmae_is_busy(sh_chan)) 179fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return -EBUSY; 180d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1818b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski sh_chan->xmit_shift = calc_xmit_shift(sh_chan, val); 182d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu sh_dmae_writel(sh_chan, val, CHCR); 183cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 184d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return 0; 185d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 186d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 187d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic int dmae_set_dmars(struct sh_dmae_chan *sh_chan, u16 val) 188d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 189027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct sh_dmae_device *shdev = container_of(sh_chan->common.device, 190027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct sh_dmae_device, common); 191027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct sh_dmae_pdata *pdata = shdev->pdata; 192027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct sh_dmae_channel *chan_pdata = &pdata->channel[sh_chan->id]; 193027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski u16 __iomem *addr = shdev->dmars + chan_pdata->dmars / sizeof(u16); 194027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski int shift = chan_pdata->dmars_bit; 195fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 196fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (dmae_is_busy(sh_chan)) 197fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return -EBUSY; 198d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 199027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski __raw_writew((__raw_readw(addr) & (0xff00 >> shift)) | (val << shift), 200027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski addr); 201d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 202d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return 0; 203d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 204d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 205d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic dma_cookie_t sh_dmae_tx_submit(struct dma_async_tx_descriptor *tx) 206d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 2073542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski struct sh_desc *desc = tx_to_sh_desc(tx), *chunk, *last = desc, *c; 208d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_chan *sh_chan = to_sh_chan(tx->chan); 2093542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dma_async_tx_callback callback = tx->callback; 210d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_cookie_t cookie; 211d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 212d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_lock_bh(&sh_chan->desc_lock); 213d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 214d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu cookie = sh_chan->common.cookie; 215d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu cookie++; 216d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (cookie < 0) 217d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu cookie = 1; 218d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 2193542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski sh_chan->common.cookie = cookie; 2203542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski tx->cookie = cookie; 2213542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 2223542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Mark all chunks of this descriptor as submitted, move to the queue */ 2233542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski list_for_each_entry_safe(chunk, c, desc->node.prev, node) { 2243542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* 2253542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * All chunks are on the global ld_free, so, we have to find 2263542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * the end of the chain ourselves 2273542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski */ 2283542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (chunk != desc && (chunk->mark == DESC_IDLE || 2293542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski chunk->async_tx.cookie > 0 || 2303542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski chunk->async_tx.cookie == -EBUSY || 2313542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski &chunk->node == &sh_chan->ld_free)) 2323542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski break; 2333542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski chunk->mark = DESC_SUBMITTED; 2343542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Callback goes to the last chunk */ 2353542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski chunk->async_tx.callback = NULL; 2363542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski chunk->cookie = cookie; 2373542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski list_move_tail(&chunk->node, &sh_chan->ld_queue); 2383542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski last = chunk; 2393542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 240d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 2413542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski last->async_tx.callback = callback; 2423542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski last->async_tx.callback_param = tx->callback_param; 2433542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 2443542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dev_dbg(sh_chan->dev, "submit #%d@%p on %d: %x[%d] -> %x\n", 2453542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski tx->cookie, &last->async_tx, sh_chan->id, 2463542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->hw.sar, desc->hw.tcr, desc->hw.dar); 247d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 248d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_unlock_bh(&sh_chan->desc_lock); 249d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 250d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return cookie; 251d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 252d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 2533542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski/* Called with desc_lock held */ 254d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic struct sh_desc *sh_dmae_get_desc(struct sh_dmae_chan *sh_chan) 255d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 2563542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski struct sh_desc *desc; 257d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 2583542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski list_for_each_entry(desc, &sh_chan->ld_free, node) 2593542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (desc->mark != DESC_PREPARED) { 2603542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski BUG_ON(desc->mark != DESC_IDLE); 261d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu list_del(&desc->node); 2623542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski return desc; 263d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 264d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 2653542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski return NULL; 266d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 267d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 268cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetskistatic struct sh_dmae_slave_config *sh_dmae_find_slave( 2694bab9d426e6dbd9ea09330919a33d35d5faab400Magnus Damm struct sh_dmae_chan *sh_chan, struct sh_dmae_slave *param) 270cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski{ 271cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct dma_device *dma_dev = sh_chan->common.device; 272cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct sh_dmae_device *shdev = container_of(dma_dev, 273cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct sh_dmae_device, common); 274027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct sh_dmae_pdata *pdata = shdev->pdata; 275cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski int i; 276cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 2774bab9d426e6dbd9ea09330919a33d35d5faab400Magnus Damm if (param->slave_id >= SHDMA_SLAVE_NUMBER) 278cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski return NULL; 279cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 280027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski for (i = 0; i < pdata->slave_num; i++) 2814bab9d426e6dbd9ea09330919a33d35d5faab400Magnus Damm if (pdata->slave[i].slave_id == param->slave_id) 282027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski return pdata->slave + i; 283cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 284cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski return NULL; 285cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski} 286cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 287d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic int sh_dmae_alloc_chan_resources(struct dma_chan *chan) 288d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 289d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_chan *sh_chan = to_sh_chan(chan); 290d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_desc *desc; 291cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct sh_dmae_slave *param = chan->private; 292cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 29320f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski pm_runtime_get_sync(sh_chan->dev); 29420f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski 295cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski /* 296cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski * This relies on the guarantee from dmaengine that alloc_chan_resources 297cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski * never runs concurrently with itself or free_chan_resources. 298cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski */ 299cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski if (param) { 300cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct sh_dmae_slave_config *cfg; 301cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 3024bab9d426e6dbd9ea09330919a33d35d5faab400Magnus Damm cfg = sh_dmae_find_slave(sh_chan, param); 303cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski if (!cfg) 304cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski return -EINVAL; 305cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 306cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski if (test_and_set_bit(param->slave_id, sh_dmae_slave_used)) 307cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski return -EBUSY; 308cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 309cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski param->config = cfg; 310cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 311cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski dmae_set_dmars(sh_chan, cfg->mid_rid); 312cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski dmae_set_chcr(sh_chan, cfg->chcr); 3138b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski } else if ((sh_dmae_readl(sh_chan, CHCR) & 0xf00) != 0x400) { 3148b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski dmae_init(sh_chan); 315cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski } 316d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 317d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_lock_bh(&sh_chan->desc_lock); 318d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu while (sh_chan->descs_allocated < NR_DESCS_PER_CHANNEL) { 319d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_unlock_bh(&sh_chan->desc_lock); 320d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu desc = kzalloc(sizeof(struct sh_desc), GFP_KERNEL); 321d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (!desc) { 322d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_lock_bh(&sh_chan->desc_lock); 323d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu break; 324d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 325d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_async_tx_descriptor_init(&desc->async_tx, 326d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu &sh_chan->common); 327d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu desc->async_tx.tx_submit = sh_dmae_tx_submit; 3283542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark = DESC_IDLE; 329d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 330d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_lock_bh(&sh_chan->desc_lock); 3313542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski list_add(&desc->node, &sh_chan->ld_free); 332d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu sh_chan->descs_allocated++; 333d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 334d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_unlock_bh(&sh_chan->desc_lock); 335d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 33620f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski if (!sh_chan->descs_allocated) 33720f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski pm_runtime_put(sh_chan->dev); 33820f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski 339d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return sh_chan->descs_allocated; 340d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 341d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 342d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu/* 343d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * sh_dma_free_chan_resources - Free all resources of the channel. 344d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu */ 345d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void sh_dmae_free_chan_resources(struct dma_chan *chan) 346d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 347d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_chan *sh_chan = to_sh_chan(chan); 348d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_desc *desc, *_desc; 349d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu LIST_HEAD(list); 35020f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski int descs = sh_chan->descs_allocated; 351d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 352cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski dmae_halt(sh_chan); 353cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 3543542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Prepared and not submitted descriptors can still be on the queue */ 3553542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (!list_empty(&sh_chan->ld_queue)) 3563542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski sh_dmae_chan_ld_cleanup(sh_chan, true); 3573542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 358cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski if (chan->private) { 359cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski /* The caller is holding dma_list_mutex */ 360cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct sh_dmae_slave *param = chan->private; 361cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski clear_bit(param->slave_id, sh_dmae_slave_used); 362cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski } 363cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 364d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_lock_bh(&sh_chan->desc_lock); 365d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 366d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu list_splice_init(&sh_chan->ld_free, &list); 367d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu sh_chan->descs_allocated = 0; 368d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 369d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_unlock_bh(&sh_chan->desc_lock); 370d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 37120f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski if (descs > 0) 37220f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski pm_runtime_put(sh_chan->dev); 37320f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski 374d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu list_for_each_entry_safe(desc, _desc, &list, node) 375d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu kfree(desc); 376d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 377d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 378cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski/** 379fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * sh_dmae_add_desc - get, set up and return one transfer descriptor 380fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * @sh_chan: DMA channel 381fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * @flags: DMA transfer flags 382fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * @dest: destination DMA address, incremented when direction equals 383fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * DMA_FROM_DEVICE or DMA_BIDIRECTIONAL 384fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * @src: source DMA address, incremented when direction equals 385fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * DMA_TO_DEVICE or DMA_BIDIRECTIONAL 386fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * @len: DMA transfer length 387fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * @first: if NULL, set to the current descriptor and cookie set to -EBUSY 388fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * @direction: needed for slave DMA to decide which address to keep constant, 389fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * equals DMA_BIDIRECTIONAL for MEMCPY 390fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * Returns 0 or an error 391fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * Locks: called with desc_lock held 392fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski */ 393fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetskistatic struct sh_desc *sh_dmae_add_desc(struct sh_dmae_chan *sh_chan, 394fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski unsigned long flags, dma_addr_t *dest, dma_addr_t *src, size_t *len, 395fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct sh_desc **first, enum dma_data_direction direction) 396d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 397fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct sh_desc *new; 398d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu size_t copy_size; 399d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 400fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (!*len) 401d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return NULL; 402d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 403fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski /* Allocate the link descriptor from the free list */ 404fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new = sh_dmae_get_desc(sh_chan); 405fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (!new) { 406fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski dev_err(sh_chan->dev, "No free link descriptor available\n"); 407d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return NULL; 408fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski } 409d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 410fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski copy_size = min(*len, (size_t)SH_DMA_TCR_MAX + 1); 411fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 412fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->hw.sar = *src; 413fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->hw.dar = *dest; 414fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->hw.tcr = copy_size; 415fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 416fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (!*first) { 417fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski /* First desc */ 418fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->async_tx.cookie = -EBUSY; 419fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski *first = new; 420fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski } else { 421fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski /* Other desc - invisible to the user */ 422fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->async_tx.cookie = -EINVAL; 423fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski } 424fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 425cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski dev_dbg(sh_chan->dev, 426cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski "chaining (%u/%u)@%x -> %x with %p, cookie %d, shift %d\n", 427fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski copy_size, *len, *src, *dest, &new->async_tx, 428cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski new->async_tx.cookie, sh_chan->xmit_shift); 429fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 430fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->mark = DESC_PREPARED; 431fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->async_tx.flags = flags; 432cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski new->direction = direction; 433fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 434fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski *len -= copy_size; 435fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (direction == DMA_BIDIRECTIONAL || direction == DMA_TO_DEVICE) 436fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski *src += copy_size; 437fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (direction == DMA_BIDIRECTIONAL || direction == DMA_FROM_DEVICE) 438fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski *dest += copy_size; 439fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 440fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return new; 441fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski} 442fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 443fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski/* 444fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * sh_dmae_prep_sg - prepare transfer descriptors from an SG list 445fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * 446fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * Common routine for public (MEMCPY) and slave DMA. The MEMCPY case is also 447fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * converted to scatter-gather to guarantee consistent locking and a correct 448fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * list manipulation. For slave DMA direction carries the usual meaning, and, 449fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * logically, the SG list is RAM and the addr variable contains slave address, 450fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * e.g., the FIFO I/O register. For MEMCPY direction equals DMA_BIDIRECTIONAL 451fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * and the SG list contains only one element and points at the source buffer. 452fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski */ 453fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetskistatic struct dma_async_tx_descriptor *sh_dmae_prep_sg(struct sh_dmae_chan *sh_chan, 454fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct scatterlist *sgl, unsigned int sg_len, dma_addr_t *addr, 455fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski enum dma_data_direction direction, unsigned long flags) 456fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski{ 457fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct scatterlist *sg; 458fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct sh_desc *first = NULL, *new = NULL /* compiler... */; 459fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski LIST_HEAD(tx_list); 460fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski int chunks = 0; 461fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski int i; 462fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 463fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (!sg_len) 464fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return NULL; 465fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 466fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski for_each_sg(sgl, sg, sg_len, i) 467fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski chunks += (sg_dma_len(sg) + SH_DMA_TCR_MAX) / 468fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski (SH_DMA_TCR_MAX + 1); 469d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 4703542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Have to lock the whole loop to protect against concurrent release */ 4713542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski spin_lock_bh(&sh_chan->desc_lock); 4723542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 4733542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* 4743542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * Chaining: 4753542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * first descriptor is what user is dealing with in all API calls, its 4763542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * cookie is at first set to -EBUSY, at tx-submit to a positive 4773542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * number 4783542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * if more than one chunk is needed further chunks have cookie = -EINVAL 4793542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * the last chunk, if not equal to the first, has cookie = -ENOSPC 4803542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * all chunks are linked onto the tx_list head with their .node heads 4813542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * only during this function, then they are immediately spliced 4823542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * back onto the free list in form of a chain 4833542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski */ 484fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski for_each_sg(sgl, sg, sg_len, i) { 485fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski dma_addr_t sg_addr = sg_dma_address(sg); 486fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski size_t len = sg_dma_len(sg); 487fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 488fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (!len) 489fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski goto err_get_desc; 490fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 491fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski do { 492fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski dev_dbg(sh_chan->dev, "Add SG #%d@%p[%d], dma %llx\n", 493fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski i, sg, len, (unsigned long long)sg_addr); 494fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 495fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (direction == DMA_FROM_DEVICE) 496fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new = sh_dmae_add_desc(sh_chan, flags, 497fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski &sg_addr, addr, &len, &first, 498fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski direction); 499fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski else 500fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new = sh_dmae_add_desc(sh_chan, flags, 501fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski addr, &sg_addr, &len, &first, 502fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski direction); 503fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (!new) 504fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski goto err_get_desc; 505fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 506fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->chunks = chunks--; 507fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski list_add_tail(&new->node, &tx_list); 508fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski } while (len); 509fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski } 510d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 5113542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (new != first) 5123542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski new->async_tx.cookie = -ENOSPC; 513d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 5143542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Put them back on the free list, so, they don't get lost */ 5153542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski list_splice_tail(&tx_list, &sh_chan->ld_free); 516d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 5173542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski spin_unlock_bh(&sh_chan->desc_lock); 518d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 5193542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski return &first->async_tx; 520fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 521fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetskierr_get_desc: 522fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski list_for_each_entry(new, &tx_list, node) 523fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->mark = DESC_IDLE; 524fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski list_splice(&tx_list, &sh_chan->ld_free); 525fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 526fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski spin_unlock_bh(&sh_chan->desc_lock); 527fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 528fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return NULL; 529fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski} 530fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 531fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetskistatic struct dma_async_tx_descriptor *sh_dmae_prep_memcpy( 532fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct dma_chan *chan, dma_addr_t dma_dest, dma_addr_t dma_src, 533fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski size_t len, unsigned long flags) 534fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski{ 535fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct sh_dmae_chan *sh_chan; 536fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct scatterlist sg; 537fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 538fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (!chan || !len) 539fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return NULL; 540fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 541cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski chan->private = NULL; 542cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 543fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski sh_chan = to_sh_chan(chan); 544fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 545fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski sg_init_table(&sg, 1); 546fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski sg_set_page(&sg, pfn_to_page(PFN_DOWN(dma_src)), len, 547fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski offset_in_page(dma_src)); 548fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski sg_dma_address(&sg) = dma_src; 549fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski sg_dma_len(&sg) = len; 550fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 551fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return sh_dmae_prep_sg(sh_chan, &sg, 1, &dma_dest, DMA_BIDIRECTIONAL, 552fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski flags); 553d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 554d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 555cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetskistatic struct dma_async_tx_descriptor *sh_dmae_prep_slave_sg( 556cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, 557cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski enum dma_data_direction direction, unsigned long flags) 558cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski{ 559cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct sh_dmae_slave *param; 560cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct sh_dmae_chan *sh_chan; 561cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 562cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski if (!chan) 563cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski return NULL; 564cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 565cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski sh_chan = to_sh_chan(chan); 566cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski param = chan->private; 567cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 568cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski /* Someone calling slave DMA on a public channel? */ 569cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski if (!param || !sg_len) { 570cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski dev_warn(sh_chan->dev, "%s: bad parameter: %p, %d, %d\n", 571cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski __func__, param, sg_len, param ? param->slave_id : -1); 572cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski return NULL; 573cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski } 574cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 575cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski /* 576cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski * if (param != NULL), this is a successfully requested slave channel, 577cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski * therefore param->config != NULL too. 578cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski */ 579cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski return sh_dmae_prep_sg(sh_chan, sgl, sg_len, ¶m->config->addr, 580cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski direction, flags); 581cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski} 582cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 583cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetskistatic void sh_dmae_terminate_all(struct dma_chan *chan) 584cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski{ 585cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct sh_dmae_chan *sh_chan = to_sh_chan(chan); 586cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 587cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski if (!chan) 588cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski return; 589cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 590c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski dmae_halt(sh_chan); 591c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski 592c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski spin_lock_bh(&sh_chan->desc_lock); 593c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski if (!list_empty(&sh_chan->ld_queue)) { 594c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski /* Record partial transfer */ 595c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski struct sh_desc *desc = list_entry(sh_chan->ld_queue.next, 596c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski struct sh_desc, node); 597c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski desc->partial = (desc->hw.tcr - sh_dmae_readl(sh_chan, TCR)) << 598c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski sh_chan->xmit_shift; 599c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski 600c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski } 601c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski spin_unlock_bh(&sh_chan->desc_lock); 602c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski 603cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski sh_dmae_chan_ld_cleanup(sh_chan, true); 604cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski} 605cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 6063542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetskistatic dma_async_tx_callback __ld_cleanup(struct sh_dmae_chan *sh_chan, bool all) 607d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 608d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_desc *desc, *_desc; 6093542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Is the "exposed" head of a chain acked? */ 6103542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski bool head_acked = false; 6113542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dma_cookie_t cookie = 0; 6123542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dma_async_tx_callback callback = NULL; 6133542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski void *param = NULL; 614d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 615d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_lock_bh(&sh_chan->desc_lock); 616d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu list_for_each_entry_safe(desc, _desc, &sh_chan->ld_queue, node) { 6173542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski struct dma_async_tx_descriptor *tx = &desc->async_tx; 6183542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 6193542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski BUG_ON(tx->cookie > 0 && tx->cookie != desc->cookie); 6203542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski BUG_ON(desc->mark != DESC_SUBMITTED && 6213542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark != DESC_COMPLETED && 6223542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark != DESC_WAITING); 6233542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 6243542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* 6253542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * queue is ordered, and we use this loop to (1) clean up all 6263542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * completed descriptors, and to (2) update descriptor flags of 6273542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * any chunks in a (partially) completed chain 6283542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski */ 6293542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (!all && desc->mark == DESC_SUBMITTED && 6303542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->cookie != cookie) 631d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu break; 632d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 6333542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (tx->cookie > 0) 6343542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski cookie = tx->cookie; 635d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 6363542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (desc->mark == DESC_COMPLETED && desc->chunks == 1) { 637cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski if (sh_chan->completed_cookie != desc->cookie - 1) 638cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski dev_dbg(sh_chan->dev, 639cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski "Completing cookie %d, expected %d\n", 640cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski desc->cookie, 641cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski sh_chan->completed_cookie + 1); 6423542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski sh_chan->completed_cookie = desc->cookie; 6433542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 644d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 6453542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Call callback on the last chunk */ 6463542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (desc->mark == DESC_COMPLETED && tx->callback) { 6473542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark = DESC_WAITING; 6483542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski callback = tx->callback; 6493542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski param = tx->callback_param; 6503542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dev_dbg(sh_chan->dev, "descriptor #%d@%p on %d callback\n", 6513542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski tx->cookie, tx, sh_chan->id); 6523542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski BUG_ON(desc->chunks != 1); 6533542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski break; 6543542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 655d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 6563542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (tx->cookie > 0 || tx->cookie == -EBUSY) { 6573542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (desc->mark == DESC_COMPLETED) { 6583542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski BUG_ON(tx->cookie < 0); 6593542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark = DESC_WAITING; 6603542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 6613542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski head_acked = async_tx_test_ack(tx); 6623542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } else { 6633542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski switch (desc->mark) { 6643542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski case DESC_COMPLETED: 6653542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark = DESC_WAITING; 6663542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Fall through */ 6673542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski case DESC_WAITING: 6683542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (head_acked) 6693542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski async_tx_ack(&desc->async_tx); 6703542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 6713542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 6723542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 6733542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dev_dbg(sh_chan->dev, "descriptor %p #%d completed.\n", 6743542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski tx, tx->cookie); 6753542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 6763542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (((desc->mark == DESC_COMPLETED || 6773542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark == DESC_WAITING) && 6783542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski async_tx_test_ack(&desc->async_tx)) || all) { 6793542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Remove from ld_queue list */ 6803542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark = DESC_IDLE; 6813542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski list_move(&desc->node, &sh_chan->ld_free); 682d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 683d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 684d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_unlock_bh(&sh_chan->desc_lock); 6853542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 6863542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (callback) 6873542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski callback(param); 6883542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 6893542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski return callback; 6903542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski} 6913542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 6923542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski/* 6933542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * sh_chan_ld_cleanup - Clean up link descriptors 6943542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * 6953542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * This function cleans up the ld_queue of DMA channel. 6963542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski */ 6973542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetskistatic void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan, bool all) 6983542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski{ 6993542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski while (__ld_cleanup(sh_chan, all)) 7003542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski ; 701d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 702d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 703d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan) 704d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 70547a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski struct sh_desc *desc; 706d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 7073542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski spin_lock_bh(&sh_chan->desc_lock); 708d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* DMA work check */ 7093542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (dmae_is_busy(sh_chan)) { 7103542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski spin_unlock_bh(&sh_chan->desc_lock); 711d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return; 7123542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 713d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 714cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski /* Find the first not transferred desciptor */ 71547a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski list_for_each_entry(desc, &sh_chan->ld_queue, node) 71647a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski if (desc->mark == DESC_SUBMITTED) { 717c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski dev_dbg(sh_chan->dev, "Queue #%d to %d: %u@%x -> %x\n", 718c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski desc->async_tx.cookie, sh_chan->id, 719c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski desc->hw.tcr, desc->hw.sar, desc->hw.dar); 7203542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Get the ld start address from ld_queue */ 72147a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski dmae_set_reg(sh_chan, &desc->hw); 7223542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dmae_start(sh_chan); 7233542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski break; 7243542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 7253542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 7263542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski spin_unlock_bh(&sh_chan->desc_lock); 727d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 728d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 729d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void sh_dmae_memcpy_issue_pending(struct dma_chan *chan) 730d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 731d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_chan *sh_chan = to_sh_chan(chan); 732d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu sh_chan_xfer_ld_queue(sh_chan); 733d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 734d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 735d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic enum dma_status sh_dmae_is_complete(struct dma_chan *chan, 736d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_cookie_t cookie, 737d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_cookie_t *done, 738d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_cookie_t *used) 739d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 740d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_chan *sh_chan = to_sh_chan(chan); 741d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_cookie_t last_used; 742d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_cookie_t last_complete; 74347a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski enum dma_status status; 744d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 7453542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski sh_dmae_chan_ld_cleanup(sh_chan, false); 746d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 747d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu last_used = chan->cookie; 748d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu last_complete = sh_chan->completed_cookie; 7493542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski BUG_ON(last_complete < 0); 750d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 751d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (done) 752d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu *done = last_complete; 753d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 754d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (used) 755d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu *used = last_used; 756d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 75747a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski spin_lock_bh(&sh_chan->desc_lock); 75847a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski 75947a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski status = dma_async_is_complete(cookie, last_complete, last_used); 76047a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski 76147a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski /* 76247a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski * If we don't find cookie on the queue, it has been aborted and we have 76347a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski * to report error 76447a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski */ 76547a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski if (status != DMA_SUCCESS) { 76647a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski struct sh_desc *desc; 76747a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski status = DMA_ERROR; 76847a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski list_for_each_entry(desc, &sh_chan->ld_queue, node) 76947a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski if (desc->cookie == cookie) { 77047a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski status = DMA_IN_PROGRESS; 77147a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski break; 77247a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski } 77347a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski } 77447a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski 77547a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski spin_unlock_bh(&sh_chan->desc_lock); 77647a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski 77747a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski return status; 778d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 779d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 780d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic irqreturn_t sh_dmae_interrupt(int irq, void *data) 781d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 782d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu irqreturn_t ret = IRQ_NONE; 783d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_chan *sh_chan = (struct sh_dmae_chan *)data; 784d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu u32 chcr = sh_dmae_readl(sh_chan, CHCR); 785d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 786d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (chcr & CHCR_TE) { 787d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* DMA stop */ 788d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dmae_halt(sh_chan); 789d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 790d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu ret = IRQ_HANDLED; 791d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu tasklet_schedule(&sh_chan->tasklet); 792d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 793d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 794d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return ret; 795d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 796d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 797d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#if defined(CONFIG_CPU_SH4) 798d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic irqreturn_t sh_dmae_err(int irq, void *data) 799d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 800d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_device *shdev = (struct sh_dmae_device *)data; 80147a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski int i; 802d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 80347a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski /* halt the dma controller */ 804027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski sh_dmae_ctl_stop(shdev); 80547a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski 80647a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski /* We cannot detect, which channel caused the error, have to reset all */ 8078b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski for (i = 0; i < SH_DMAC_MAX_CHANNELS; i++) { 80847a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski struct sh_dmae_chan *sh_chan = shdev->chan[i]; 80947a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski if (sh_chan) { 81047a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski struct sh_desc *desc; 81147a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski /* Stop the channel */ 81247a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski dmae_halt(sh_chan); 81347a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski /* Complete all */ 81447a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski list_for_each_entry(desc, &sh_chan->ld_queue, node) { 81547a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski struct dma_async_tx_descriptor *tx = &desc->async_tx; 81647a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski desc->mark = DESC_IDLE; 81747a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski if (tx->callback) 81847a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski tx->callback(tx->callback_param); 819d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 82047a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski list_splice_init(&sh_chan->ld_queue, &sh_chan->ld_free); 821d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 822d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 823027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski sh_dmae_rst(shdev); 82447a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski 82547a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski return IRQ_HANDLED; 826d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 827d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#endif 828d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 829d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void dmae_do_tasklet(unsigned long data) 830d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 831d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_chan *sh_chan = (struct sh_dmae_chan *)data; 8323542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski struct sh_desc *desc; 833d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu u32 sar_buf = sh_dmae_readl(sh_chan, SAR); 834cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski u32 dar_buf = sh_dmae_readl(sh_chan, DAR); 83586d61b33e48f1da5a6b310d3de93187db62ab72aGuennadi Liakhovetski 8363542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski spin_lock(&sh_chan->desc_lock); 8373542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski list_for_each_entry(desc, &sh_chan->ld_queue, node) { 838cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski if (desc->mark == DESC_SUBMITTED && 839cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski ((desc->direction == DMA_FROM_DEVICE && 840cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski (desc->hw.dar + desc->hw.tcr) == dar_buf) || 841cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski (desc->hw.sar + desc->hw.tcr) == sar_buf)) { 8423542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dev_dbg(sh_chan->dev, "done #%d@%p dst %u\n", 8433542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->async_tx.cookie, &desc->async_tx, 8443542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->hw.dar); 8453542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark = DESC_COMPLETED; 846d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu break; 847d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 848d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 8493542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski spin_unlock(&sh_chan->desc_lock); 850d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 851d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* Next desc */ 852d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu sh_chan_xfer_ld_queue(sh_chan); 8533542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski sh_dmae_chan_ld_cleanup(sh_chan, false); 854d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 855d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 856027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskistatic int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id, 857027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski int irq, unsigned long flags) 858d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 859d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu int err; 860027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct sh_dmae_channel *chan_pdata = &shdev->pdata->channel[id]; 861027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct platform_device *pdev = to_platform_device(shdev->common.dev); 862d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_chan *new_sh_chan; 863d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 864d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* alloc channel */ 865d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu new_sh_chan = kzalloc(sizeof(struct sh_dmae_chan), GFP_KERNEL); 866d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (!new_sh_chan) { 86786d61b33e48f1da5a6b310d3de93187db62ab72aGuennadi Liakhovetski dev_err(shdev->common.dev, 86886d61b33e48f1da5a6b310d3de93187db62ab72aGuennadi Liakhovetski "No free memory for allocating dma channels!\n"); 869d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return -ENOMEM; 870d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 871d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 8728b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski /* copy struct dma_device */ 8738b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski new_sh_chan->common.device = &shdev->common; 8748b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski 875d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu new_sh_chan->dev = shdev->common.dev; 876d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu new_sh_chan->id = id; 877027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski new_sh_chan->irq = irq; 878027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski new_sh_chan->base = shdev->chan_reg + chan_pdata->offset / sizeof(u32); 879d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 880d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* Init DMA tasklet */ 881d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu tasklet_init(&new_sh_chan->tasklet, dmae_do_tasklet, 882d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu (unsigned long)new_sh_chan); 883d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 884d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* Init the channel */ 885d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dmae_init(new_sh_chan); 886d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 887d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_lock_init(&new_sh_chan->desc_lock); 888d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 889d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* Init descripter manage list */ 890d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu INIT_LIST_HEAD(&new_sh_chan->ld_queue); 891d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu INIT_LIST_HEAD(&new_sh_chan->ld_free); 892d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 893d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* Add the channel to DMA device channel list */ 894d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu list_add_tail(&new_sh_chan->common.device_node, 895d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu &shdev->common.channels); 896d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->common.chancnt++; 897d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 898027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (pdev->id >= 0) 899027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski snprintf(new_sh_chan->dev_id, sizeof(new_sh_chan->dev_id), 900027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski "sh-dmae%d.%d", pdev->id, new_sh_chan->id); 901027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski else 902027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski snprintf(new_sh_chan->dev_id, sizeof(new_sh_chan->dev_id), 903027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski "sh-dma%d", new_sh_chan->id); 904d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 905d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* set up channel irq */ 906027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski err = request_irq(irq, &sh_dmae_interrupt, flags, 90786d61b33e48f1da5a6b310d3de93187db62ab72aGuennadi Liakhovetski new_sh_chan->dev_id, new_sh_chan); 908d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (err) { 909d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dev_err(shdev->common.dev, "DMA channel %d request_irq error " 910d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu "with return %d\n", id, err); 911d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu goto err_no_irq; 912d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 913d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 914d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->chan[id] = new_sh_chan; 915d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return 0; 916d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 917d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsuerr_no_irq: 918d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* remove from dmaengine device node */ 919d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu list_del(&new_sh_chan->common.device_node); 920d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu kfree(new_sh_chan); 921d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return err; 922d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 923d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 924d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void sh_dmae_chan_remove(struct sh_dmae_device *shdev) 925d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 926d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu int i; 927d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 928d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu for (i = shdev->common.chancnt - 1 ; i >= 0 ; i--) { 929d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (shdev->chan[i]) { 930027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct sh_dmae_chan *sh_chan = shdev->chan[i]; 931027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 932027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski free_irq(sh_chan->irq, sh_chan); 933d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 934027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski list_del(&sh_chan->common.device_node); 935027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski kfree(sh_chan); 936d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->chan[i] = NULL; 937d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 938d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 939d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->common.chancnt = 0; 940d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 941d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 942d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic int __init sh_dmae_probe(struct platform_device *pdev) 943d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 944027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct sh_dmae_pdata *pdata = pdev->dev.platform_data; 945027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski unsigned long irqflags = IRQF_DISABLED, 9468b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski chan_flag[SH_DMAC_MAX_CHANNELS] = {}; 9478b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski int errirq, chan_irq[SH_DMAC_MAX_CHANNELS]; 948027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski int err, i, irq_cnt = 0, irqres = 0; 949d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_device *shdev; 950027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct resource *chan, *dmars, *errirq_res, *chanirq_res; 951d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 95256adf7e8127d601b172e180b44551ce83404348fDan Williams /* get platform data */ 953027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (!pdata || !pdata->channel_num) 95456adf7e8127d601b172e180b44551ce83404348fDan Williams return -ENODEV; 95556adf7e8127d601b172e180b44551ce83404348fDan Williams 956027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chan = platform_get_resource(pdev, IORESOURCE_MEM, 0); 957027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski /* DMARS area is optional, if absent, this controller cannot do slave DMA */ 958027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dmars = platform_get_resource(pdev, IORESOURCE_MEM, 1); 959027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski /* 960027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * IRQ resources: 961027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * 1. there always must be at least one IRQ IO-resource. On SH4 it is 962027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * the error IRQ, in which case it is the only IRQ in this resource: 963027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * start == end. If it is the only IRQ resource, all channels also 964027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * use the same IRQ. 965027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * 2. DMA channel IRQ resources can be specified one per resource or in 966027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * ranges (start != end) 967027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * 3. iff all events (channels and, optionally, error) on this 968027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * controller use the same IRQ, only one IRQ resource can be 969027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * specified, otherwise there must be one IRQ per channel, even if 970027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * some of them are equal 971027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * 4. if all IRQs on this controller are equal or if some specific IRQs 972027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * specify IORESOURCE_IRQ_SHAREABLE in their resources, they will be 973027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * requested with the IRQF_SHARED flag 974027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski */ 975027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski errirq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); 976027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (!chan || !errirq_res) 977027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski return -ENODEV; 978027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 979027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (!request_mem_region(chan->start, resource_size(chan), pdev->name)) { 980027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dev_err(&pdev->dev, "DMAC register region already claimed\n"); 981027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski return -EBUSY; 982027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski } 983027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 984027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (dmars && !request_mem_region(dmars->start, resource_size(dmars), pdev->name)) { 985027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dev_err(&pdev->dev, "DMAC DMARS region already claimed\n"); 986027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski err = -EBUSY; 987027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski goto ermrdmars; 988027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski } 989027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 990027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski err = -ENOMEM; 991d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev = kzalloc(sizeof(struct sh_dmae_device), GFP_KERNEL); 992d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (!shdev) { 993027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dev_err(&pdev->dev, "Not enough memory\n"); 994027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski goto ealloc; 995027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski } 996027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 997027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski shdev->chan_reg = ioremap(chan->start, resource_size(chan)); 998027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (!shdev->chan_reg) 999027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski goto emapchan; 1000027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (dmars) { 1001027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski shdev->dmars = ioremap(dmars->start, resource_size(dmars)); 1002027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (!shdev->dmars) 1003027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski goto emapdmars; 1004d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1005d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1006d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* platform data */ 1007027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski shdev->pdata = pdata; 1008d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 100920f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski pm_runtime_enable(&pdev->dev); 101020f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski pm_runtime_get_sync(&pdev->dev); 101120f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski 1012d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* reset dma controller */ 1013027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski err = sh_dmae_rst(shdev); 1014d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (err) 1015d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu goto rst_err; 1016d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1017d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu INIT_LIST_HEAD(&shdev->common.channels); 1018d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1019d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_cap_set(DMA_MEMCPY, shdev->common.cap_mask); 1020027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (dmars) 1021027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dma_cap_set(DMA_SLAVE, shdev->common.cap_mask); 1022cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 1023d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->common.device_alloc_chan_resources 1024d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu = sh_dmae_alloc_chan_resources; 1025d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->common.device_free_chan_resources = sh_dmae_free_chan_resources; 1026d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->common.device_prep_dma_memcpy = sh_dmae_prep_memcpy; 1027d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->common.device_is_tx_complete = sh_dmae_is_complete; 1028d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->common.device_issue_pending = sh_dmae_memcpy_issue_pending; 1029cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 1030cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski /* Compulsory for DMA_SLAVE fields */ 1031cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski shdev->common.device_prep_slave_sg = sh_dmae_prep_slave_sg; 1032cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski shdev->common.device_terminate_all = sh_dmae_terminate_all; 1033cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 1034d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->common.dev = &pdev->dev; 1035ddb4f0f0e05871c7ac540cc778993c06ff53b765Guennadi Liakhovetski /* Default transfer size of 32 bytes requires 32-byte alignment */ 10368b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski shdev->common.copy_align = LOG2_DEFAULT_XFER_SIZE; 1037d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1038d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#if defined(CONFIG_CPU_SH4) 1039027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chanirq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 1); 1040027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1041027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (!chanirq_res) 1042027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chanirq_res = errirq_res; 1043027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski else 1044027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski irqres++; 1045027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1046027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (chanirq_res == errirq_res || 1047027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski (errirq_res->flags & IORESOURCE_BITS) == IORESOURCE_IRQ_SHAREABLE) 1048d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu irqflags = IRQF_SHARED; 1049027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1050027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski errirq = errirq_res->start; 1051027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1052027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski err = request_irq(errirq, sh_dmae_err, irqflags, 1053027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski "DMAC Address Error", shdev); 1054027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (err) { 1055027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dev_err(&pdev->dev, 1056027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski "DMA failed requesting irq #%d, error %d\n", 1057027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski errirq, err); 1058027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski goto eirq_err; 1059d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1060d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1061027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski#else 1062027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chanirq_res = errirq_res; 1063027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski#endif /* CONFIG_CPU_SH4 */ 1064027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1065027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (chanirq_res->start == chanirq_res->end && 1066027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski !platform_get_resource(pdev, IORESOURCE_IRQ, 1)) { 1067027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski /* Special case - all multiplexed */ 1068027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski for (; irq_cnt < pdata->channel_num; irq_cnt++) { 1069027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chan_irq[irq_cnt] = chanirq_res->start; 1070027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chan_flag[irq_cnt] = IRQF_SHARED; 1071d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1072027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski } else { 1073027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski do { 1074027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski for (i = chanirq_res->start; i <= chanirq_res->end; i++) { 1075027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if ((errirq_res->flags & IORESOURCE_BITS) == 1076027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski IORESOURCE_IRQ_SHAREABLE) 1077027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chan_flag[irq_cnt] = IRQF_SHARED; 1078027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski else 1079027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chan_flag[irq_cnt] = IRQF_DISABLED; 1080027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dev_dbg(&pdev->dev, 1081027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski "Found IRQ %d for channel %d\n", 1082027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski i, irq_cnt); 1083027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chan_irq[irq_cnt++] = i; 1084027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski } 1085027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chanirq_res = platform_get_resource(pdev, 1086027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski IORESOURCE_IRQ, ++irqres); 1087027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski } while (irq_cnt < pdata->channel_num && chanirq_res); 1088d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1089027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1090027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (irq_cnt < pdata->channel_num) 1091027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski goto eirqres; 1092d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1093d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* Create DMA Channel */ 1094027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski for (i = 0; i < pdata->channel_num; i++) { 1095027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski err = sh_dmae_chan_probe(shdev, i, chan_irq[i], chan_flag[i]); 1096d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (err) 1097d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu goto chan_probe_err; 1098d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1099d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 110020f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski pm_runtime_put(&pdev->dev); 110120f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski 1102d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu platform_set_drvdata(pdev, shdev); 1103d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_async_device_register(&shdev->common); 1104d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1105d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return err; 1106d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1107d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsuchan_probe_err: 1108d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu sh_dmae_chan_remove(shdev); 1109027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskieirqres: 1110027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski#if defined(CONFIG_CPU_SH4) 1111027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski free_irq(errirq, shdev); 1112d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsueirq_err: 1113027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski#endif 1114d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsurst_err: 111520f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski pm_runtime_put(&pdev->dev); 1116027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (dmars) 1117027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski iounmap(shdev->dmars); 1118027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskiemapdmars: 1119027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski iounmap(shdev->chan_reg); 1120027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskiemapchan: 1121d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu kfree(shdev); 1122027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskiealloc: 1123027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (dmars) 1124027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski release_mem_region(dmars->start, resource_size(dmars)); 1125027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskiermrdmars: 1126027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski release_mem_region(chan->start, resource_size(chan)); 1127d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1128d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return err; 1129d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 1130d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1131d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic int __exit sh_dmae_remove(struct platform_device *pdev) 1132d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1133d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_device *shdev = platform_get_drvdata(pdev); 1134027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct resource *res; 1135027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski int errirq = platform_get_irq(pdev, 0); 1136d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1137d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_async_device_unregister(&shdev->common); 1138d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1139027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (errirq > 0) 1140027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski free_irq(errirq, shdev); 1141d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1142d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* channel data remove */ 1143d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu sh_dmae_chan_remove(shdev); 1144d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 114520f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski pm_runtime_disable(&pdev->dev); 114620f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski 1147027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (shdev->dmars) 1148027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski iounmap(shdev->dmars); 1149027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski iounmap(shdev->chan_reg); 1150027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1151d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu kfree(shdev); 1152d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1153027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1154027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (res) 1155027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski release_mem_region(res->start, resource_size(res)); 1156027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski res = platform_get_resource(pdev, IORESOURCE_MEM, 1); 1157027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (res) 1158027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski release_mem_region(res->start, resource_size(res)); 1159027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1160d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return 0; 1161d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 1162d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1163d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void sh_dmae_shutdown(struct platform_device *pdev) 1164d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1165d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_device *shdev = platform_get_drvdata(pdev); 1166027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski sh_dmae_ctl_stop(shdev); 1167d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 1168d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1169d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic struct platform_driver sh_dmae_driver = { 1170d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu .remove = __exit_p(sh_dmae_remove), 1171d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu .shutdown = sh_dmae_shutdown, 1172d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu .driver = { 1173d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu .name = "sh-dma-engine", 1174d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu }, 1175d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu}; 1176d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1177d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic int __init sh_dmae_init(void) 1178d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1179d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return platform_driver_probe(&sh_dmae_driver, sh_dmae_probe); 1180d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 1181d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsumodule_init(sh_dmae_init); 1182d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1183d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void __exit sh_dmae_exit(void) 1184d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1185d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu platform_driver_unregister(&sh_dmae_driver); 1186d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 1187d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsumodule_exit(sh_dmae_exit); 1188d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1189d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro IwamatsuMODULE_AUTHOR("Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com>"); 1190d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro IwamatsuMODULE_DESCRIPTION("Renesas SH DMA Engine driver"); 1191d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro IwamatsuMODULE_LICENSE("GPL"); 1192