shdma.c revision 300e5f97d2a32196cbe03104cd6ffe2af97d9338
1d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu/* 2d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * Renesas SuperH DMA Engine support 3d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * 4d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * base is drivers/dma/flsdma.c 5d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * 6d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * Copyright (C) 2009 Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com> 7d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * Copyright (C) 2009 Renesas Solutions, Inc. All rights reserved. 8d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved. 9d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * 10d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * This is free software; you can redistribute it and/or modify 11d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * it under the terms of the GNU General Public License as published by 12d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * the Free Software Foundation; either version 2 of the License, or 13d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * (at your option) any later version. 14d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * 15d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * - DMA of SuperH does not have Hardware DMA chain mode. 16d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * - MAX DMA size is 16MB. 17d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * 18d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu */ 19d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 20d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include <linux/init.h> 21d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include <linux/module.h> 225a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 23d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include <linux/interrupt.h> 24d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include <linux/dmaengine.h> 25d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include <linux/delay.h> 26d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include <linux/dma-mapping.h> 27d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include <linux/platform_device.h> 2820f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski#include <linux/pm_runtime.h> 29b2623a61cfd3c6badb8396dc85ab5a70f4a05f61Magnus Damm#include <linux/sh_dma.h> 3003aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt#include <linux/notifier.h> 3103aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt#include <linux/kdebug.h> 3203aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt#include <linux/spinlock.h> 3303aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt#include <linux/rculist.h> 34d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#include "shdma.h" 35d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 36d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu/* DMA descriptor control */ 373542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetskienum sh_dmae_desc_status { 383542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski DESC_IDLE, 393542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski DESC_PREPARED, 403542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski DESC_SUBMITTED, 413542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski DESC_COMPLETED, /* completed, have to call callback */ 423542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski DESC_WAITING, /* callback called, waiting for ack / re-submit */ 433542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski}; 44d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 45d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu#define NR_DESCS_PER_CHANNEL 32 468b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski/* Default MEMCPY transfer size = 2^2 = 4 bytes */ 478b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski#define LOG2_DEFAULT_XFER_SIZE 2 48d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 4903aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt/* 5003aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt * Used for write-side mutual exclusion for the global device list, 512dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski * read-side synchronization by way of RCU, and per-controller data. 5203aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt */ 5303aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundtstatic DEFINE_SPINLOCK(sh_dmae_lock); 5403aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundtstatic LIST_HEAD(sh_dmae_devices); 5503aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 56cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski/* A bitmask with bits enough for enum sh_dmae_slave_chan_id */ 5702ca5083f60521d09f13224596564a405108bc4cMagnus Dammstatic unsigned long sh_dmae_slave_used[BITS_TO_LONGS(SH_DMA_SLAVE_NUMBER)]; 58cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 593542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetskistatic void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan, bool all); 603542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 61d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void sh_dmae_writel(struct sh_dmae_chan *sh_dc, u32 data, u32 reg) 62d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 63027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski __raw_writel(data, sh_dc->base + reg / sizeof(u32)); 64d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 65d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 66d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic u32 sh_dmae_readl(struct sh_dmae_chan *sh_dc, u32 reg) 67d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 68027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski return __raw_readl(sh_dc->base + reg / sizeof(u32)); 69027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski} 70027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 71027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskistatic u16 dmaor_read(struct sh_dmae_device *shdev) 72027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski{ 73027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski return __raw_readw(shdev->chan_reg + DMAOR / sizeof(u32)); 74027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski} 75027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 76027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskistatic void dmaor_write(struct sh_dmae_device *shdev, u16 data) 77027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski{ 78027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski __raw_writew(data, shdev->chan_reg + DMAOR / sizeof(u32)); 79d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 80d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 81d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu/* 82d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * Reset DMA controller 83d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * 84d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * SH7780 has two DMAOR register 85d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu */ 86027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskistatic void sh_dmae_ctl_stop(struct sh_dmae_device *shdev) 87d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 882dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski unsigned short dmaor; 892dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski unsigned long flags; 902dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 912dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_lock_irqsave(&sh_dmae_lock, flags); 92d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 932dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski dmaor = dmaor_read(shdev); 94027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dmaor_write(shdev, dmaor & ~(DMAOR_NMIF | DMAOR_AE | DMAOR_DME)); 952dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 962dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_unlock_irqrestore(&sh_dmae_lock, flags); 97d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 98d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 99027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskistatic int sh_dmae_rst(struct sh_dmae_device *shdev) 100d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 101d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu unsigned short dmaor; 1022dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski unsigned long flags; 103d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1042dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_lock_irqsave(&sh_dmae_lock, flags); 105d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1062dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski dmaor = dmaor_read(shdev) & ~(DMAOR_NMIF | DMAOR_AE | DMAOR_DME); 1072dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 1082dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski dmaor_write(shdev, dmaor | shdev->pdata->dmaor_init); 1092dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 1102dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski dmaor = dmaor_read(shdev); 1112dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 1122dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_unlock_irqrestore(&sh_dmae_lock, flags); 1132dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 1142dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski if (dmaor & (DMAOR_AE | DMAOR_NMIF)) { 1152dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski dev_warn(shdev->common.dev, "Can't initialize DMAOR.\n"); 1162dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski return -EIO; 117d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 118d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return 0; 119d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 120d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 121fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetskistatic bool dmae_is_busy(struct sh_dmae_chan *sh_chan) 122d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 123d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu u32 chcr = sh_dmae_readl(sh_chan, CHCR); 124fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 125fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if ((chcr & (CHCR_DE | CHCR_TE)) == CHCR_DE) 126fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return true; /* working */ 127fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 128fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return false; /* waiting */ 129d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 130d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1318b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetskistatic unsigned int calc_xmit_shift(struct sh_dmae_chan *sh_chan, u32 chcr) 132d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1338b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski struct sh_dmae_device *shdev = container_of(sh_chan->common.device, 1348b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski struct sh_dmae_device, common); 1358b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski struct sh_dmae_pdata *pdata = shdev->pdata; 1368b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski int cnt = ((chcr & pdata->ts_low_mask) >> pdata->ts_low_shift) | 1378b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski ((chcr & pdata->ts_high_mask) >> pdata->ts_high_shift); 1388b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski 1398b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski if (cnt >= pdata->ts_shift_num) 1408b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski cnt = 0; 141623b4ac4bf9e767991c66e29b47dd4b19458fb42Guennadi Liakhovetski 1428b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski return pdata->ts_shift[cnt]; 1438b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski} 1448b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski 1458b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetskistatic u32 log2size_to_chcr(struct sh_dmae_chan *sh_chan, int l2size) 1468b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski{ 1478b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski struct sh_dmae_device *shdev = container_of(sh_chan->common.device, 1488b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski struct sh_dmae_device, common); 1498b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski struct sh_dmae_pdata *pdata = shdev->pdata; 1508b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski int i; 1518b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski 1528b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski for (i = 0; i < pdata->ts_shift_num; i++) 1538b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski if (pdata->ts_shift[i] == l2size) 1548b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski break; 1558b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski 1568b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski if (i == pdata->ts_shift_num) 1578b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski i = 0; 1588b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski 1598b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski return ((i << pdata->ts_low_shift) & pdata->ts_low_mask) | 1608b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski ((i << pdata->ts_high_shift) & pdata->ts_high_mask); 161d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 162d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1633542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetskistatic void dmae_set_reg(struct sh_dmae_chan *sh_chan, struct sh_dmae_regs *hw) 164d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1653542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski sh_dmae_writel(sh_chan, hw->sar, SAR); 1663542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski sh_dmae_writel(sh_chan, hw->dar, DAR); 167cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski sh_dmae_writel(sh_chan, hw->tcr >> sh_chan->xmit_shift, TCR); 168d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 169d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 170d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void dmae_start(struct sh_dmae_chan *sh_chan) 171d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 172d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu u32 chcr = sh_dmae_readl(sh_chan, CHCR); 173d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 17486d61b33e48f1da5a6b310d3de93187db62ab72aGuennadi Liakhovetski chcr |= CHCR_DE | CHCR_IE; 175cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski sh_dmae_writel(sh_chan, chcr & ~CHCR_TE, CHCR); 176d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 177d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 178d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void dmae_halt(struct sh_dmae_chan *sh_chan) 179d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 180d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu u32 chcr = sh_dmae_readl(sh_chan, CHCR); 181d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 182d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu chcr &= ~(CHCR_DE | CHCR_TE | CHCR_IE); 183d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu sh_dmae_writel(sh_chan, chcr, CHCR); 184d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 185d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 186cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetskistatic void dmae_init(struct sh_dmae_chan *sh_chan) 187cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski{ 1888b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski /* 1898b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski * Default configuration for dual address memory-memory transfer. 1908b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski * 0x400 represents auto-request. 1918b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski */ 1928b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski u32 chcr = DM_INC | SM_INC | 0x400 | log2size_to_chcr(sh_chan, 1938b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski LOG2_DEFAULT_XFER_SIZE); 1948b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski sh_chan->xmit_shift = calc_xmit_shift(sh_chan, chcr); 195cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski sh_dmae_writel(sh_chan, chcr, CHCR); 196cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski} 197cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 198d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic int dmae_set_chcr(struct sh_dmae_chan *sh_chan, u32 val) 199d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 2002dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski /* If DMA is active, cannot set CHCR. TODO: remove this superfluous check */ 201fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (dmae_is_busy(sh_chan)) 202fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return -EBUSY; 203d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 2048b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski sh_chan->xmit_shift = calc_xmit_shift(sh_chan, val); 205d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu sh_dmae_writel(sh_chan, val, CHCR); 206cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 207d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return 0; 208d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 209d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 210d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic int dmae_set_dmars(struct sh_dmae_chan *sh_chan, u16 val) 211d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 212027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct sh_dmae_device *shdev = container_of(sh_chan->common.device, 213027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct sh_dmae_device, common); 214027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct sh_dmae_pdata *pdata = shdev->pdata; 2155bac942db3d2c4738df04104240d65a5d1eaec6aGuennadi Liakhovetski const struct sh_dmae_channel *chan_pdata = &pdata->channel[sh_chan->id]; 21626fc02ab5551349b2e593829a76cb44328ee7f61Magnus Damm u16 __iomem *addr = shdev->dmars; 217027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski int shift = chan_pdata->dmars_bit; 218fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 219fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (dmae_is_busy(sh_chan)) 220fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return -EBUSY; 221d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 22226fc02ab5551349b2e593829a76cb44328ee7f61Magnus Damm /* in the case of a missing DMARS resource use first memory window */ 22326fc02ab5551349b2e593829a76cb44328ee7f61Magnus Damm if (!addr) 22426fc02ab5551349b2e593829a76cb44328ee7f61Magnus Damm addr = (u16 __iomem *)shdev->chan_reg; 22526fc02ab5551349b2e593829a76cb44328ee7f61Magnus Damm addr += chan_pdata->dmars / sizeof(u16); 22626fc02ab5551349b2e593829a76cb44328ee7f61Magnus Damm 227027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski __raw_writew((__raw_readw(addr) & (0xff00 >> shift)) | (val << shift), 228027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski addr); 229d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 230d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return 0; 231d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 232d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 233d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic dma_cookie_t sh_dmae_tx_submit(struct dma_async_tx_descriptor *tx) 234d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 2353542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski struct sh_desc *desc = tx_to_sh_desc(tx), *chunk, *last = desc, *c; 236d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_chan *sh_chan = to_sh_chan(tx->chan); 2373542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dma_async_tx_callback callback = tx->callback; 238d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_cookie_t cookie; 239d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 240d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_lock_bh(&sh_chan->desc_lock); 241d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 242d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu cookie = sh_chan->common.cookie; 243d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu cookie++; 244d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (cookie < 0) 245d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu cookie = 1; 246d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 2473542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski sh_chan->common.cookie = cookie; 2483542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski tx->cookie = cookie; 2493542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 2503542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Mark all chunks of this descriptor as submitted, move to the queue */ 2513542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski list_for_each_entry_safe(chunk, c, desc->node.prev, node) { 2523542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* 2533542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * All chunks are on the global ld_free, so, we have to find 2543542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * the end of the chain ourselves 2553542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski */ 2563542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (chunk != desc && (chunk->mark == DESC_IDLE || 2573542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski chunk->async_tx.cookie > 0 || 2583542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski chunk->async_tx.cookie == -EBUSY || 2593542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski &chunk->node == &sh_chan->ld_free)) 2603542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski break; 2613542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski chunk->mark = DESC_SUBMITTED; 2623542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Callback goes to the last chunk */ 2633542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski chunk->async_tx.callback = NULL; 2643542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski chunk->cookie = cookie; 2653542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski list_move_tail(&chunk->node, &sh_chan->ld_queue); 2663542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski last = chunk; 2673542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 268d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 2693542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski last->async_tx.callback = callback; 2703542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski last->async_tx.callback_param = tx->callback_param; 2713542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 2723542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dev_dbg(sh_chan->dev, "submit #%d@%p on %d: %x[%d] -> %x\n", 2733542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski tx->cookie, &last->async_tx, sh_chan->id, 2743542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->hw.sar, desc->hw.tcr, desc->hw.dar); 275d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 276d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_unlock_bh(&sh_chan->desc_lock); 277d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 278d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return cookie; 279d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 280d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 2813542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski/* Called with desc_lock held */ 282d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic struct sh_desc *sh_dmae_get_desc(struct sh_dmae_chan *sh_chan) 283d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 2843542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski struct sh_desc *desc; 285d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 2863542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski list_for_each_entry(desc, &sh_chan->ld_free, node) 2873542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (desc->mark != DESC_PREPARED) { 2883542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski BUG_ON(desc->mark != DESC_IDLE); 289d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu list_del(&desc->node); 2903542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski return desc; 291d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 292d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 2933542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski return NULL; 294d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 295d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 2965bac942db3d2c4738df04104240d65a5d1eaec6aGuennadi Liakhovetskistatic const struct sh_dmae_slave_config *sh_dmae_find_slave( 2974bab9d426e6dbd9ea09330919a33d35d5faab400Magnus Damm struct sh_dmae_chan *sh_chan, struct sh_dmae_slave *param) 298cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski{ 299cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct dma_device *dma_dev = sh_chan->common.device; 300cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct sh_dmae_device *shdev = container_of(dma_dev, 301cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct sh_dmae_device, common); 302027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct sh_dmae_pdata *pdata = shdev->pdata; 303cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski int i; 304cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 30502ca5083f60521d09f13224596564a405108bc4cMagnus Damm if (param->slave_id >= SH_DMA_SLAVE_NUMBER) 306cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski return NULL; 307cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 308027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski for (i = 0; i < pdata->slave_num; i++) 3094bab9d426e6dbd9ea09330919a33d35d5faab400Magnus Damm if (pdata->slave[i].slave_id == param->slave_id) 310027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski return pdata->slave + i; 311cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 312cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski return NULL; 313cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski} 314cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 315d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic int sh_dmae_alloc_chan_resources(struct dma_chan *chan) 316d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 317d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_chan *sh_chan = to_sh_chan(chan); 318d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_desc *desc; 319cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct sh_dmae_slave *param = chan->private; 32083515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski int ret; 321cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 32220f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski pm_runtime_get_sync(sh_chan->dev); 32320f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski 324cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski /* 325cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski * This relies on the guarantee from dmaengine that alloc_chan_resources 326cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski * never runs concurrently with itself or free_chan_resources. 327cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski */ 328cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski if (param) { 3295bac942db3d2c4738df04104240d65a5d1eaec6aGuennadi Liakhovetski const struct sh_dmae_slave_config *cfg; 330cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 3314bab9d426e6dbd9ea09330919a33d35d5faab400Magnus Damm cfg = sh_dmae_find_slave(sh_chan, param); 33283515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski if (!cfg) { 33383515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski ret = -EINVAL; 33483515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski goto efindslave; 33583515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski } 336cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 33783515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski if (test_and_set_bit(param->slave_id, sh_dmae_slave_used)) { 33883515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski ret = -EBUSY; 33983515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski goto etestused; 34083515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski } 341cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 342cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski param->config = cfg; 343cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 344cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski dmae_set_dmars(sh_chan, cfg->mid_rid); 345cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski dmae_set_chcr(sh_chan, cfg->chcr); 3468b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski } else if ((sh_dmae_readl(sh_chan, CHCR) & 0xf00) != 0x400) { 3478b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski dmae_init(sh_chan); 348cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski } 349d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 350d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_lock_bh(&sh_chan->desc_lock); 351d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu while (sh_chan->descs_allocated < NR_DESCS_PER_CHANNEL) { 352d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_unlock_bh(&sh_chan->desc_lock); 353d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu desc = kzalloc(sizeof(struct sh_desc), GFP_KERNEL); 354d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (!desc) { 355d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_lock_bh(&sh_chan->desc_lock); 356d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu break; 357d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 358d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_async_tx_descriptor_init(&desc->async_tx, 359d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu &sh_chan->common); 360d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu desc->async_tx.tx_submit = sh_dmae_tx_submit; 3613542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark = DESC_IDLE; 362d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 363d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_lock_bh(&sh_chan->desc_lock); 3643542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski list_add(&desc->node, &sh_chan->ld_free); 365d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu sh_chan->descs_allocated++; 366d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 367d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_unlock_bh(&sh_chan->desc_lock); 368d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 36983515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski if (!sh_chan->descs_allocated) { 37083515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski ret = -ENOMEM; 37183515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski goto edescalloc; 37283515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski } 37320f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski 374d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return sh_chan->descs_allocated; 37583515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski 37683515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetskiedescalloc: 37783515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski if (param) 37883515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski clear_bit(param->slave_id, sh_dmae_slave_used); 37983515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetskietestused: 38083515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetskiefindslave: 38183515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski pm_runtime_put(sh_chan->dev); 38283515bc7df812555e20cda48614674e2f346f9f5Guennadi Liakhovetski return ret; 383d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 384d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 385d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu/* 386d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu * sh_dma_free_chan_resources - Free all resources of the channel. 387d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu */ 388d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void sh_dmae_free_chan_resources(struct dma_chan *chan) 389d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 390d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_chan *sh_chan = to_sh_chan(chan); 391d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_desc *desc, *_desc; 392d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu LIST_HEAD(list); 39320f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski int descs = sh_chan->descs_allocated; 394d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 3952dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski /* Protect against ISR */ 3962dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_lock_irq(&sh_chan->desc_lock); 397cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski dmae_halt(sh_chan); 3982dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_unlock_irq(&sh_chan->desc_lock); 3992dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 4002dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski /* Now no new interrupts will occur */ 401cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 4023542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Prepared and not submitted descriptors can still be on the queue */ 4033542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (!list_empty(&sh_chan->ld_queue)) 4043542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski sh_dmae_chan_ld_cleanup(sh_chan, true); 4053542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 406cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski if (chan->private) { 407cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski /* The caller is holding dma_list_mutex */ 408cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct sh_dmae_slave *param = chan->private; 409cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski clear_bit(param->slave_id, sh_dmae_slave_used); 4102dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski chan->private = NULL; 411cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski } 412cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 413d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_lock_bh(&sh_chan->desc_lock); 414d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 415d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu list_splice_init(&sh_chan->ld_free, &list); 416d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu sh_chan->descs_allocated = 0; 417d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 418d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_unlock_bh(&sh_chan->desc_lock); 419d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 42020f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski if (descs > 0) 42120f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski pm_runtime_put(sh_chan->dev); 42220f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski 423d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu list_for_each_entry_safe(desc, _desc, &list, node) 424d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu kfree(desc); 425d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 426d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 427cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski/** 428fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * sh_dmae_add_desc - get, set up and return one transfer descriptor 429fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * @sh_chan: DMA channel 430fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * @flags: DMA transfer flags 431fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * @dest: destination DMA address, incremented when direction equals 432fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * DMA_FROM_DEVICE or DMA_BIDIRECTIONAL 433fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * @src: source DMA address, incremented when direction equals 434fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * DMA_TO_DEVICE or DMA_BIDIRECTIONAL 435fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * @len: DMA transfer length 436fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * @first: if NULL, set to the current descriptor and cookie set to -EBUSY 437fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * @direction: needed for slave DMA to decide which address to keep constant, 438fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * equals DMA_BIDIRECTIONAL for MEMCPY 439fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * Returns 0 or an error 440fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * Locks: called with desc_lock held 441fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski */ 442fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetskistatic struct sh_desc *sh_dmae_add_desc(struct sh_dmae_chan *sh_chan, 443fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski unsigned long flags, dma_addr_t *dest, dma_addr_t *src, size_t *len, 444fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct sh_desc **first, enum dma_data_direction direction) 445d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 446fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct sh_desc *new; 447d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu size_t copy_size; 448d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 449fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (!*len) 450d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return NULL; 451d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 452fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski /* Allocate the link descriptor from the free list */ 453fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new = sh_dmae_get_desc(sh_chan); 454fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (!new) { 455fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski dev_err(sh_chan->dev, "No free link descriptor available\n"); 456d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return NULL; 457fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski } 458d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 459fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski copy_size = min(*len, (size_t)SH_DMA_TCR_MAX + 1); 460fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 461fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->hw.sar = *src; 462fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->hw.dar = *dest; 463fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->hw.tcr = copy_size; 464fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 465fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (!*first) { 466fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski /* First desc */ 467fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->async_tx.cookie = -EBUSY; 468fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski *first = new; 469fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski } else { 470fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski /* Other desc - invisible to the user */ 471fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->async_tx.cookie = -EINVAL; 472fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski } 473fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 474cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski dev_dbg(sh_chan->dev, 475cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski "chaining (%u/%u)@%x -> %x with %p, cookie %d, shift %d\n", 476fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski copy_size, *len, *src, *dest, &new->async_tx, 477cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski new->async_tx.cookie, sh_chan->xmit_shift); 478fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 479fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->mark = DESC_PREPARED; 480fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->async_tx.flags = flags; 481cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski new->direction = direction; 482fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 483fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski *len -= copy_size; 484fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (direction == DMA_BIDIRECTIONAL || direction == DMA_TO_DEVICE) 485fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski *src += copy_size; 486fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (direction == DMA_BIDIRECTIONAL || direction == DMA_FROM_DEVICE) 487fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski *dest += copy_size; 488fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 489fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return new; 490fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski} 491fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 492fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski/* 493fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * sh_dmae_prep_sg - prepare transfer descriptors from an SG list 494fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * 495fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * Common routine for public (MEMCPY) and slave DMA. The MEMCPY case is also 496fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * converted to scatter-gather to guarantee consistent locking and a correct 497fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * list manipulation. For slave DMA direction carries the usual meaning, and, 498fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * logically, the SG list is RAM and the addr variable contains slave address, 499fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * e.g., the FIFO I/O register. For MEMCPY direction equals DMA_BIDIRECTIONAL 500fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski * and the SG list contains only one element and points at the source buffer. 501fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski */ 502fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetskistatic struct dma_async_tx_descriptor *sh_dmae_prep_sg(struct sh_dmae_chan *sh_chan, 503fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct scatterlist *sgl, unsigned int sg_len, dma_addr_t *addr, 504fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski enum dma_data_direction direction, unsigned long flags) 505fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski{ 506fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct scatterlist *sg; 507fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct sh_desc *first = NULL, *new = NULL /* compiler... */; 508fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski LIST_HEAD(tx_list); 509fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski int chunks = 0; 510fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski int i; 511fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 512fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (!sg_len) 513fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return NULL; 514fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 515fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski for_each_sg(sgl, sg, sg_len, i) 516fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski chunks += (sg_dma_len(sg) + SH_DMA_TCR_MAX) / 517fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski (SH_DMA_TCR_MAX + 1); 518d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 5193542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Have to lock the whole loop to protect against concurrent release */ 5203542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski spin_lock_bh(&sh_chan->desc_lock); 5213542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 5223542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* 5233542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * Chaining: 5243542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * first descriptor is what user is dealing with in all API calls, its 5253542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * cookie is at first set to -EBUSY, at tx-submit to a positive 5263542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * number 5273542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * if more than one chunk is needed further chunks have cookie = -EINVAL 5283542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * the last chunk, if not equal to the first, has cookie = -ENOSPC 5293542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * all chunks are linked onto the tx_list head with their .node heads 5303542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * only during this function, then they are immediately spliced 5313542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * back onto the free list in form of a chain 5323542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski */ 533fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski for_each_sg(sgl, sg, sg_len, i) { 534fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski dma_addr_t sg_addr = sg_dma_address(sg); 535fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski size_t len = sg_dma_len(sg); 536fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 537fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (!len) 538fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski goto err_get_desc; 539fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 540fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski do { 541fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski dev_dbg(sh_chan->dev, "Add SG #%d@%p[%d], dma %llx\n", 542fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski i, sg, len, (unsigned long long)sg_addr); 543fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 544fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (direction == DMA_FROM_DEVICE) 545fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new = sh_dmae_add_desc(sh_chan, flags, 546fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski &sg_addr, addr, &len, &first, 547fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski direction); 548fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski else 549fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new = sh_dmae_add_desc(sh_chan, flags, 550fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski addr, &sg_addr, &len, &first, 551fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski direction); 552fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (!new) 553fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski goto err_get_desc; 554fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 555fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->chunks = chunks--; 556fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski list_add_tail(&new->node, &tx_list); 557fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski } while (len); 558fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski } 559d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 5603542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (new != first) 5613542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski new->async_tx.cookie = -ENOSPC; 562d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 5633542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Put them back on the free list, so, they don't get lost */ 5643542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski list_splice_tail(&tx_list, &sh_chan->ld_free); 565d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 5663542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski spin_unlock_bh(&sh_chan->desc_lock); 567d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 5683542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski return &first->async_tx; 569fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 570fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetskierr_get_desc: 571fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski list_for_each_entry(new, &tx_list, node) 572fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski new->mark = DESC_IDLE; 573fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski list_splice(&tx_list, &sh_chan->ld_free); 574fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 575fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski spin_unlock_bh(&sh_chan->desc_lock); 576fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 577fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return NULL; 578fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski} 579fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 580fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetskistatic struct dma_async_tx_descriptor *sh_dmae_prep_memcpy( 581fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct dma_chan *chan, dma_addr_t dma_dest, dma_addr_t dma_src, 582fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski size_t len, unsigned long flags) 583fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski{ 584fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct sh_dmae_chan *sh_chan; 585fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski struct scatterlist sg; 586fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 587fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski if (!chan || !len) 588fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return NULL; 589fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 590fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski sh_chan = to_sh_chan(chan); 591fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 592fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski sg_init_table(&sg, 1); 593fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski sg_set_page(&sg, pfn_to_page(PFN_DOWN(dma_src)), len, 594fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski offset_in_page(dma_src)); 595fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski sg_dma_address(&sg) = dma_src; 596fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski sg_dma_len(&sg) = len; 597fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski 598fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski return sh_dmae_prep_sg(sh_chan, &sg, 1, &dma_dest, DMA_BIDIRECTIONAL, 599fc4618575f79eea062cdc51715040e40cd35b71cGuennadi Liakhovetski flags); 600d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 601d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 602cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetskistatic struct dma_async_tx_descriptor *sh_dmae_prep_slave_sg( 603cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, 604cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski enum dma_data_direction direction, unsigned long flags) 605cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski{ 606cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct sh_dmae_slave *param; 607cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct sh_dmae_chan *sh_chan; 6085bac942db3d2c4738df04104240d65a5d1eaec6aGuennadi Liakhovetski dma_addr_t slave_addr; 609cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 610cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski if (!chan) 611cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski return NULL; 612cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 613cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski sh_chan = to_sh_chan(chan); 614cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski param = chan->private; 615cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 616cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski /* Someone calling slave DMA on a public channel? */ 617cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski if (!param || !sg_len) { 618cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski dev_warn(sh_chan->dev, "%s: bad parameter: %p, %d, %d\n", 619cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski __func__, param, sg_len, param ? param->slave_id : -1); 620cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski return NULL; 621cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski } 622cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 6239f9ff20d46c6728b092f34b6a642e1e81ab5e254Dan Carpenter slave_addr = param->config->addr; 6249f9ff20d46c6728b092f34b6a642e1e81ab5e254Dan Carpenter 625cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski /* 626cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski * if (param != NULL), this is a successfully requested slave channel, 627cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski * therefore param->config != NULL too. 628cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski */ 6295bac942db3d2c4738df04104240d65a5d1eaec6aGuennadi Liakhovetski return sh_dmae_prep_sg(sh_chan, sgl, sg_len, &slave_addr, 630cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski direction, flags); 631cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski} 632cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 633058276303dbc4ed089c1f7dad0871810b1f5ddf1Linus Walleijstatic int sh_dmae_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, 634058276303dbc4ed089c1f7dad0871810b1f5ddf1Linus Walleij unsigned long arg) 635cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski{ 636cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski struct sh_dmae_chan *sh_chan = to_sh_chan(chan); 637cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 638c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij /* Only supports DMA_TERMINATE_ALL */ 639c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij if (cmd != DMA_TERMINATE_ALL) 640c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij return -ENXIO; 641c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij 642cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski if (!chan) 643c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij return -EINVAL; 644cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 6452dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_lock_bh(&sh_chan->desc_lock); 646c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski dmae_halt(sh_chan); 647c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski 648c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski if (!list_empty(&sh_chan->ld_queue)) { 649c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski /* Record partial transfer */ 650c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski struct sh_desc *desc = list_entry(sh_chan->ld_queue.next, 651c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski struct sh_desc, node); 652c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski desc->partial = (desc->hw.tcr - sh_dmae_readl(sh_chan, TCR)) << 653c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski sh_chan->xmit_shift; 654c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski 655c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski } 656c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski spin_unlock_bh(&sh_chan->desc_lock); 657c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski 658cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski sh_dmae_chan_ld_cleanup(sh_chan, true); 659c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij 660c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij return 0; 661cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski} 662cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 6633542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetskistatic dma_async_tx_callback __ld_cleanup(struct sh_dmae_chan *sh_chan, bool all) 664d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 665d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_desc *desc, *_desc; 6663542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Is the "exposed" head of a chain acked? */ 6673542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski bool head_acked = false; 6683542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dma_cookie_t cookie = 0; 6693542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dma_async_tx_callback callback = NULL; 6703542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski void *param = NULL; 671d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 672d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_lock_bh(&sh_chan->desc_lock); 673d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu list_for_each_entry_safe(desc, _desc, &sh_chan->ld_queue, node) { 6743542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski struct dma_async_tx_descriptor *tx = &desc->async_tx; 6753542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 6763542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski BUG_ON(tx->cookie > 0 && tx->cookie != desc->cookie); 6773542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski BUG_ON(desc->mark != DESC_SUBMITTED && 6783542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark != DESC_COMPLETED && 6793542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark != DESC_WAITING); 6803542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 6813542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* 6823542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * queue is ordered, and we use this loop to (1) clean up all 6833542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * completed descriptors, and to (2) update descriptor flags of 6843542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * any chunks in a (partially) completed chain 6853542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski */ 6863542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (!all && desc->mark == DESC_SUBMITTED && 6873542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->cookie != cookie) 688d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu break; 689d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 6903542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (tx->cookie > 0) 6913542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski cookie = tx->cookie; 692d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 6933542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (desc->mark == DESC_COMPLETED && desc->chunks == 1) { 694cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski if (sh_chan->completed_cookie != desc->cookie - 1) 695cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski dev_dbg(sh_chan->dev, 696cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski "Completing cookie %d, expected %d\n", 697cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski desc->cookie, 698cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski sh_chan->completed_cookie + 1); 6993542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski sh_chan->completed_cookie = desc->cookie; 7003542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 701d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 7023542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Call callback on the last chunk */ 7033542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (desc->mark == DESC_COMPLETED && tx->callback) { 7043542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark = DESC_WAITING; 7053542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski callback = tx->callback; 7063542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski param = tx->callback_param; 7073542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dev_dbg(sh_chan->dev, "descriptor #%d@%p on %d callback\n", 7083542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski tx->cookie, tx, sh_chan->id); 7093542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski BUG_ON(desc->chunks != 1); 7103542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski break; 7113542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 712d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 7133542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (tx->cookie > 0 || tx->cookie == -EBUSY) { 7143542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (desc->mark == DESC_COMPLETED) { 7153542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski BUG_ON(tx->cookie < 0); 7163542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark = DESC_WAITING; 7173542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 7183542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski head_acked = async_tx_test_ack(tx); 7193542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } else { 7203542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski switch (desc->mark) { 7213542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski case DESC_COMPLETED: 7223542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark = DESC_WAITING; 7233542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Fall through */ 7243542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski case DESC_WAITING: 7253542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (head_acked) 7263542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski async_tx_ack(&desc->async_tx); 7273542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 7283542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 7293542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 7303542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dev_dbg(sh_chan->dev, "descriptor %p #%d completed.\n", 7313542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski tx, tx->cookie); 7323542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 7333542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (((desc->mark == DESC_COMPLETED || 7343542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark == DESC_WAITING) && 7353542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski async_tx_test_ack(&desc->async_tx)) || all) { 7363542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Remove from ld_queue list */ 7373542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark = DESC_IDLE; 7383542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski list_move(&desc->node, &sh_chan->ld_free); 739d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 740d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 7412dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 7422dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski if (all && !callback) 7432dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski /* 7442dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski * Terminating and the loop completed normally: forgive 7452dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski * uncompleted cookies 7462dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski */ 7472dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski sh_chan->completed_cookie = sh_chan->common.cookie; 7482dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 749d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_unlock_bh(&sh_chan->desc_lock); 7503542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 7513542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (callback) 7523542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski callback(param); 7533542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 7543542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski return callback; 7553542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski} 7563542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 7573542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski/* 7583542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * sh_chan_ld_cleanup - Clean up link descriptors 7593542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * 7603542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski * This function cleans up the ld_queue of DMA channel. 7613542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski */ 7623542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetskistatic void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan, bool all) 7633542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski{ 7643542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski while (__ld_cleanup(sh_chan, all)) 7653542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski ; 766d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 767d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 768d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan) 769d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 77047a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski struct sh_desc *desc; 771d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 7723542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski spin_lock_bh(&sh_chan->desc_lock); 773d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* DMA work check */ 7743542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski if (dmae_is_busy(sh_chan)) { 7753542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski spin_unlock_bh(&sh_chan->desc_lock); 776d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return; 7773542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 778d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 7795a3a7658863f74f28cef53b9336bff7423659801Justin P. Mattock /* Find the first not transferred descriptor */ 78047a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski list_for_each_entry(desc, &sh_chan->ld_queue, node) 78147a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski if (desc->mark == DESC_SUBMITTED) { 782c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski dev_dbg(sh_chan->dev, "Queue #%d to %d: %u@%x -> %x\n", 783c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski desc->async_tx.cookie, sh_chan->id, 784c014906a870ce70e009def0c9d170ccabeb0be63Guennadi Liakhovetski desc->hw.tcr, desc->hw.sar, desc->hw.dar); 7853542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski /* Get the ld start address from ld_queue */ 78647a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski dmae_set_reg(sh_chan, &desc->hw); 7873542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dmae_start(sh_chan); 7883542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski break; 7893542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski } 7903542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski 7913542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski spin_unlock_bh(&sh_chan->desc_lock); 792d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 793d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 794d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void sh_dmae_memcpy_issue_pending(struct dma_chan *chan) 795d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 796d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_chan *sh_chan = to_sh_chan(chan); 797d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu sh_chan_xfer_ld_queue(sh_chan); 798d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 799d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 8000793448187643b50af89d36b08470baf45a3cab4Linus Walleijstatic enum dma_status sh_dmae_tx_status(struct dma_chan *chan, 801d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_cookie_t cookie, 8020793448187643b50af89d36b08470baf45a3cab4Linus Walleij struct dma_tx_state *txstate) 803d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 804d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_chan *sh_chan = to_sh_chan(chan); 805d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_cookie_t last_used; 806d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_cookie_t last_complete; 80747a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski enum dma_status status; 808d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 8093542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski sh_dmae_chan_ld_cleanup(sh_chan, false); 810d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 8112dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski /* First read completed cookie to avoid a skew */ 812d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu last_complete = sh_chan->completed_cookie; 8132dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski rmb(); 8142dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski last_used = chan->cookie; 8153542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski BUG_ON(last_complete < 0); 816bca3469205402d9fb14060d255d8786ae2256640Dan Williams dma_set_tx_state(txstate, last_complete, last_used, 0); 817d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 81847a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski spin_lock_bh(&sh_chan->desc_lock); 81947a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski 82047a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski status = dma_async_is_complete(cookie, last_complete, last_used); 82147a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski 82247a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski /* 82347a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski * If we don't find cookie on the queue, it has been aborted and we have 82447a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski * to report error 82547a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski */ 82647a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski if (status != DMA_SUCCESS) { 82747a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski struct sh_desc *desc; 82847a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski status = DMA_ERROR; 82947a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski list_for_each_entry(desc, &sh_chan->ld_queue, node) 83047a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski if (desc->cookie == cookie) { 83147a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski status = DMA_IN_PROGRESS; 83247a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski break; 83347a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski } 83447a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski } 83547a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski 83647a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski spin_unlock_bh(&sh_chan->desc_lock); 83747a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski 83847a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski return status; 839d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 840d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 841d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic irqreturn_t sh_dmae_interrupt(int irq, void *data) 842d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 843d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu irqreturn_t ret = IRQ_NONE; 8442dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski struct sh_dmae_chan *sh_chan = data; 8452dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski u32 chcr; 8462dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 8472dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_lock(&sh_chan->desc_lock); 8482dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 8492dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski chcr = sh_dmae_readl(sh_chan, CHCR); 850d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 851d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (chcr & CHCR_TE) { 852d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* DMA stop */ 853d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dmae_halt(sh_chan); 854d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 855d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu ret = IRQ_HANDLED; 856d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu tasklet_schedule(&sh_chan->tasklet); 857d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 858d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 8592dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_unlock(&sh_chan->desc_lock); 8602dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 861d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return ret; 862d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 863d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 8642dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski/* Called from error IRQ or NMI */ 8652dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetskistatic bool sh_dmae_reset(struct sh_dmae_device *shdev) 866d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 86703aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt unsigned int handled = 0; 86847a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski int i; 869d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 87047a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski /* halt the dma controller */ 871027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski sh_dmae_ctl_stop(shdev); 87247a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski 87347a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski /* We cannot detect, which channel caused the error, have to reset all */ 8748b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski for (i = 0; i < SH_DMAC_MAX_CHANNELS; i++) { 87547a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski struct sh_dmae_chan *sh_chan = shdev->chan[i]; 87603aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt struct sh_desc *desc; 8772dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski LIST_HEAD(dl); 87803aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 87903aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt if (!sh_chan) 88003aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt continue; 88103aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 8822dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_lock(&sh_chan->desc_lock); 8832dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 88403aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt /* Stop the channel */ 88503aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt dmae_halt(sh_chan); 88603aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 8872dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski list_splice_init(&sh_chan->ld_queue, &dl); 8882dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 8892dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_unlock(&sh_chan->desc_lock); 8902dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 89103aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt /* Complete all */ 8922dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski list_for_each_entry(desc, &dl, node) { 89303aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt struct dma_async_tx_descriptor *tx = &desc->async_tx; 89403aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt desc->mark = DESC_IDLE; 89503aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt if (tx->callback) 89603aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt tx->callback(tx->callback_param); 897d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 89803aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 8992dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_lock(&sh_chan->desc_lock); 9002dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski list_splice(&dl, &sh_chan->ld_free); 9012dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski spin_unlock(&sh_chan->desc_lock); 9022dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 90303aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt handled++; 904d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 90503aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 906027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski sh_dmae_rst(shdev); 90747a4dc26eeb89a3746f9b1e2092602b40469640aGuennadi Liakhovetski 90803aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt return !!handled; 90903aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt} 91003aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 91103aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundtstatic irqreturn_t sh_dmae_err(int irq, void *data) 91203aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt{ 913ff7690b48ae8571d930a2621e21f6e5a41e42b6dYoshihiro Shimoda struct sh_dmae_device *shdev = data; 914ff7690b48ae8571d930a2621e21f6e5a41e42b6dYoshihiro Shimoda 9152dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski if (!(dmaor_read(shdev) & DMAOR_AE)) 916ff7690b48ae8571d930a2621e21f6e5a41e42b6dYoshihiro Shimoda return IRQ_NONE; 9172dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski 9182dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski sh_dmae_reset(data); 9192dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski return IRQ_HANDLED; 920d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 921d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 922d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void dmae_do_tasklet(unsigned long data) 923d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 924d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_chan *sh_chan = (struct sh_dmae_chan *)data; 9253542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski struct sh_desc *desc; 926d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu u32 sar_buf = sh_dmae_readl(sh_chan, SAR); 927cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski u32 dar_buf = sh_dmae_readl(sh_chan, DAR); 92886d61b33e48f1da5a6b310d3de93187db62ab72aGuennadi Liakhovetski 9293542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski spin_lock(&sh_chan->desc_lock); 9303542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski list_for_each_entry(desc, &sh_chan->ld_queue, node) { 931cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski if (desc->mark == DESC_SUBMITTED && 932cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski ((desc->direction == DMA_FROM_DEVICE && 933cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski (desc->hw.dar + desc->hw.tcr) == dar_buf) || 934cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski (desc->hw.sar + desc->hw.tcr) == sar_buf)) { 9353542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski dev_dbg(sh_chan->dev, "done #%d@%p dst %u\n", 9363542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->async_tx.cookie, &desc->async_tx, 9373542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->hw.dar); 9383542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski desc->mark = DESC_COMPLETED; 939d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu break; 940d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 941d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 9423542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski spin_unlock(&sh_chan->desc_lock); 943d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 944d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* Next desc */ 945d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu sh_chan_xfer_ld_queue(sh_chan); 9463542a113ab2f5880f1b62e5909d754250fb57d6bGuennadi Liakhovetski sh_dmae_chan_ld_cleanup(sh_chan, false); 947d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 948d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 94903aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundtstatic bool sh_dmae_nmi_notify(struct sh_dmae_device *shdev) 95003aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt{ 95103aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt /* Fast path out if NMIF is not asserted for this controller */ 95203aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt if ((dmaor_read(shdev) & DMAOR_NMIF) == 0) 95303aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt return false; 95403aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 9552dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski return sh_dmae_reset(shdev); 95603aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt} 95703aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 95803aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundtstatic int sh_dmae_nmi_handler(struct notifier_block *self, 95903aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt unsigned long cmd, void *data) 96003aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt{ 96103aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt struct sh_dmae_device *shdev; 96203aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt int ret = NOTIFY_DONE; 96303aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt bool triggered; 96403aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 96503aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt /* 96603aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt * Only concern ourselves with NMI events. 96703aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt * 96803aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt * Normally we would check the die chain value, but as this needs 96903aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt * to be architecture independent, check for NMI context instead. 97003aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt */ 97103aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt if (!in_nmi()) 97203aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt return NOTIFY_DONE; 97303aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 97403aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt rcu_read_lock(); 97503aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt list_for_each_entry_rcu(shdev, &sh_dmae_devices, node) { 97603aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt /* 97703aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt * Only stop if one of the controllers has NMIF asserted, 97803aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt * we do not want to interfere with regular address error 97903aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt * handling or NMI events that don't concern the DMACs. 98003aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt */ 98103aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt triggered = sh_dmae_nmi_notify(shdev); 98203aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt if (triggered == true) 98303aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt ret = NOTIFY_OK; 98403aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt } 98503aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt rcu_read_unlock(); 98603aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 98703aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt return ret; 98803aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt} 98903aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 99003aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundtstatic struct notifier_block sh_dmae_nmi_notifier __read_mostly = { 99103aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt .notifier_call = sh_dmae_nmi_handler, 99203aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 99303aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt /* Run before NMI debug handler and KGDB */ 99403aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt .priority = 1, 99503aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt}; 99603aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 997027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskistatic int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id, 998027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski int irq, unsigned long flags) 999d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1000d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu int err; 10015bac942db3d2c4738df04104240d65a5d1eaec6aGuennadi Liakhovetski const struct sh_dmae_channel *chan_pdata = &shdev->pdata->channel[id]; 1002027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct platform_device *pdev = to_platform_device(shdev->common.dev); 1003d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_chan *new_sh_chan; 1004d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1005d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* alloc channel */ 1006d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu new_sh_chan = kzalloc(sizeof(struct sh_dmae_chan), GFP_KERNEL); 1007d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (!new_sh_chan) { 100886d61b33e48f1da5a6b310d3de93187db62ab72aGuennadi Liakhovetski dev_err(shdev->common.dev, 100986d61b33e48f1da5a6b310d3de93187db62ab72aGuennadi Liakhovetski "No free memory for allocating dma channels!\n"); 1010d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return -ENOMEM; 1011d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1012d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 10138b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski /* copy struct dma_device */ 10148b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski new_sh_chan->common.device = &shdev->common; 10158b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski 1016d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu new_sh_chan->dev = shdev->common.dev; 1017d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu new_sh_chan->id = id; 1018027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski new_sh_chan->irq = irq; 1019027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski new_sh_chan->base = shdev->chan_reg + chan_pdata->offset / sizeof(u32); 1020d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1021d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* Init DMA tasklet */ 1022d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu tasklet_init(&new_sh_chan->tasklet, dmae_do_tasklet, 1023d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu (unsigned long)new_sh_chan); 1024d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1025d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu spin_lock_init(&new_sh_chan->desc_lock); 1026d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1027d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* Init descripter manage list */ 1028d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu INIT_LIST_HEAD(&new_sh_chan->ld_queue); 1029d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu INIT_LIST_HEAD(&new_sh_chan->ld_free); 1030d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1031d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* Add the channel to DMA device channel list */ 1032d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu list_add_tail(&new_sh_chan->common.device_node, 1033d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu &shdev->common.channels); 1034d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->common.chancnt++; 1035d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1036027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (pdev->id >= 0) 1037027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski snprintf(new_sh_chan->dev_id, sizeof(new_sh_chan->dev_id), 1038027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski "sh-dmae%d.%d", pdev->id, new_sh_chan->id); 1039027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski else 1040027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski snprintf(new_sh_chan->dev_id, sizeof(new_sh_chan->dev_id), 1041027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski "sh-dma%d", new_sh_chan->id); 1042d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1043d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* set up channel irq */ 1044027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski err = request_irq(irq, &sh_dmae_interrupt, flags, 104586d61b33e48f1da5a6b310d3de93187db62ab72aGuennadi Liakhovetski new_sh_chan->dev_id, new_sh_chan); 1046d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (err) { 1047d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dev_err(shdev->common.dev, "DMA channel %d request_irq error " 1048d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu "with return %d\n", id, err); 1049d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu goto err_no_irq; 1050d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1051d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1052d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->chan[id] = new_sh_chan; 1053d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return 0; 1054d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1055d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsuerr_no_irq: 1056d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* remove from dmaengine device node */ 1057d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu list_del(&new_sh_chan->common.device_node); 1058d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu kfree(new_sh_chan); 1059d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return err; 1060d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 1061d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1062d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void sh_dmae_chan_remove(struct sh_dmae_device *shdev) 1063d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1064d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu int i; 1065d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1066d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu for (i = shdev->common.chancnt - 1 ; i >= 0 ; i--) { 1067d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (shdev->chan[i]) { 1068027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct sh_dmae_chan *sh_chan = shdev->chan[i]; 1069027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1070027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski free_irq(sh_chan->irq, sh_chan); 1071d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1072027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski list_del(&sh_chan->common.device_node); 1073027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski kfree(sh_chan); 1074d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->chan[i] = NULL; 1075d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1076d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1077d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->common.chancnt = 0; 1078d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 1079d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1080d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic int __init sh_dmae_probe(struct platform_device *pdev) 1081d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1082027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct sh_dmae_pdata *pdata = pdev->dev.platform_data; 1083027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski unsigned long irqflags = IRQF_DISABLED, 10848b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski chan_flag[SH_DMAC_MAX_CHANNELS] = {}; 10858b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski int errirq, chan_irq[SH_DMAC_MAX_CHANNELS]; 1086300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm int err, i, irq_cnt = 0, irqres = 0, irq_cap = 0; 1087d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_device *shdev; 1088027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct resource *chan, *dmars, *errirq_res, *chanirq_res; 1089d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 109056adf7e8127d601b172e180b44551ce83404348fDan Williams /* get platform data */ 1091027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (!pdata || !pdata->channel_num) 109256adf7e8127d601b172e180b44551ce83404348fDan Williams return -ENODEV; 109356adf7e8127d601b172e180b44551ce83404348fDan Williams 1094027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chan = platform_get_resource(pdev, IORESOURCE_MEM, 0); 109526fc02ab5551349b2e593829a76cb44328ee7f61Magnus Damm /* DMARS area is optional */ 1096027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dmars = platform_get_resource(pdev, IORESOURCE_MEM, 1); 1097027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski /* 1098027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * IRQ resources: 1099027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * 1. there always must be at least one IRQ IO-resource. On SH4 it is 1100027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * the error IRQ, in which case it is the only IRQ in this resource: 1101027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * start == end. If it is the only IRQ resource, all channels also 1102027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * use the same IRQ. 1103027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * 2. DMA channel IRQ resources can be specified one per resource or in 1104027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * ranges (start != end) 1105027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * 3. iff all events (channels and, optionally, error) on this 1106027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * controller use the same IRQ, only one IRQ resource can be 1107027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * specified, otherwise there must be one IRQ per channel, even if 1108027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * some of them are equal 1109027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * 4. if all IRQs on this controller are equal or if some specific IRQs 1110027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * specify IORESOURCE_IRQ_SHAREABLE in their resources, they will be 1111027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski * requested with the IRQF_SHARED flag 1112027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski */ 1113027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski errirq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); 1114027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (!chan || !errirq_res) 1115027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski return -ENODEV; 1116027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1117027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (!request_mem_region(chan->start, resource_size(chan), pdev->name)) { 1118027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dev_err(&pdev->dev, "DMAC register region already claimed\n"); 1119027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski return -EBUSY; 1120027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski } 1121027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1122027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (dmars && !request_mem_region(dmars->start, resource_size(dmars), pdev->name)) { 1123027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dev_err(&pdev->dev, "DMAC DMARS region already claimed\n"); 1124027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski err = -EBUSY; 1125027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski goto ermrdmars; 1126027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski } 1127027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1128027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski err = -ENOMEM; 1129d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev = kzalloc(sizeof(struct sh_dmae_device), GFP_KERNEL); 1130d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (!shdev) { 1131027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dev_err(&pdev->dev, "Not enough memory\n"); 1132027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski goto ealloc; 1133027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski } 1134027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1135027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski shdev->chan_reg = ioremap(chan->start, resource_size(chan)); 1136027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (!shdev->chan_reg) 1137027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski goto emapchan; 1138027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (dmars) { 1139027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski shdev->dmars = ioremap(dmars->start, resource_size(dmars)); 1140027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (!shdev->dmars) 1141027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski goto emapdmars; 1142d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1143d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1144d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* platform data */ 1145027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski shdev->pdata = pdata; 1146d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 114720f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski pm_runtime_enable(&pdev->dev); 114820f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski pm_runtime_get_sync(&pdev->dev); 114920f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski 115031705e21f9b5a0628c043f88ff4d20488b47b8abGuennadi Liakhovetski spin_lock_irq(&sh_dmae_lock); 115103aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt list_add_tail_rcu(&shdev->node, &sh_dmae_devices); 115231705e21f9b5a0628c043f88ff4d20488b47b8abGuennadi Liakhovetski spin_unlock_irq(&sh_dmae_lock); 115303aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 11542dc666673b5a39d005579a0ef63ae69b5094e686Guennadi Liakhovetski /* reset dma controller - only needed as a test */ 1155027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski err = sh_dmae_rst(shdev); 1156d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (err) 1157d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu goto rst_err; 1158d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1159d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu INIT_LIST_HEAD(&shdev->common.channels); 1160d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1161d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_cap_set(DMA_MEMCPY, shdev->common.cap_mask); 116226fc02ab5551349b2e593829a76cb44328ee7f61Magnus Damm if (pdata->slave && pdata->slave_num) 1163027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dma_cap_set(DMA_SLAVE, shdev->common.cap_mask); 1164cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 1165d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->common.device_alloc_chan_resources 1166d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu = sh_dmae_alloc_chan_resources; 1167d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->common.device_free_chan_resources = sh_dmae_free_chan_resources; 1168d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->common.device_prep_dma_memcpy = sh_dmae_prep_memcpy; 11690793448187643b50af89d36b08470baf45a3cab4Linus Walleij shdev->common.device_tx_status = sh_dmae_tx_status; 1170d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->common.device_issue_pending = sh_dmae_memcpy_issue_pending; 1171cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 1172cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski /* Compulsory for DMA_SLAVE fields */ 1173cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski shdev->common.device_prep_slave_sg = sh_dmae_prep_slave_sg; 1174c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij shdev->common.device_control = sh_dmae_control; 1175cfefe99795251d76d92e8457f4152f532a961ec5Guennadi Liakhovetski 1176d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu shdev->common.dev = &pdev->dev; 1177ddb4f0f0e05871c7ac540cc778993c06ff53b765Guennadi Liakhovetski /* Default transfer size of 32 bytes requires 32-byte alignment */ 11788b1935e6a36b0967efc593d67ed3aebbfbc1f5b1Guennadi Liakhovetski shdev->common.copy_align = LOG2_DEFAULT_XFER_SIZE; 1179d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1180927a7c9c1793def3a55d60c926d3945528e6bf1bMagnus Damm#if defined(CONFIG_CPU_SH4) || defined(CONFIG_ARCH_SHMOBILE) 1181027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chanirq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 1); 1182027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1183027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (!chanirq_res) 1184027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chanirq_res = errirq_res; 1185027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski else 1186027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski irqres++; 1187027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1188027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (chanirq_res == errirq_res || 1189027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski (errirq_res->flags & IORESOURCE_BITS) == IORESOURCE_IRQ_SHAREABLE) 1190d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu irqflags = IRQF_SHARED; 1191027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1192027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski errirq = errirq_res->start; 1193027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1194027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski err = request_irq(errirq, sh_dmae_err, irqflags, 1195027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski "DMAC Address Error", shdev); 1196027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (err) { 1197027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dev_err(&pdev->dev, 1198027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski "DMA failed requesting irq #%d, error %d\n", 1199027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski errirq, err); 1200027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski goto eirq_err; 1201d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1202d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1203027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski#else 1204027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chanirq_res = errirq_res; 1205927a7c9c1793def3a55d60c926d3945528e6bf1bMagnus Damm#endif /* CONFIG_CPU_SH4 || CONFIG_ARCH_SHMOBILE */ 1206027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1207027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (chanirq_res->start == chanirq_res->end && 1208027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski !platform_get_resource(pdev, IORESOURCE_IRQ, 1)) { 1209027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski /* Special case - all multiplexed */ 1210027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski for (; irq_cnt < pdata->channel_num; irq_cnt++) { 1211300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm if (irq_cnt < SH_DMAC_MAX_CHANNELS) { 1212300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm chan_irq[irq_cnt] = chanirq_res->start; 1213300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm chan_flag[irq_cnt] = IRQF_SHARED; 1214300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm } else { 1215300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm irq_cap = 1; 1216300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm break; 1217300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm } 1218d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1219027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski } else { 1220027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski do { 1221027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski for (i = chanirq_res->start; i <= chanirq_res->end; i++) { 1222027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if ((errirq_res->flags & IORESOURCE_BITS) == 1223027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski IORESOURCE_IRQ_SHAREABLE) 1224027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chan_flag[irq_cnt] = IRQF_SHARED; 1225027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski else 1226027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chan_flag[irq_cnt] = IRQF_DISABLED; 1227027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski dev_dbg(&pdev->dev, 1228027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski "Found IRQ %d for channel %d\n", 1229027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski i, irq_cnt); 1230027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chan_irq[irq_cnt++] = i; 1231300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm 1232300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm if (irq_cnt >= SH_DMAC_MAX_CHANNELS) 1233300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm break; 1234300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm } 1235300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm 1236300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm if (irq_cnt >= SH_DMAC_MAX_CHANNELS) { 1237300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm irq_cap = 1; 1238300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm break; 1239027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski } 1240027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski chanirq_res = platform_get_resource(pdev, 1241027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski IORESOURCE_IRQ, ++irqres); 1242027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski } while (irq_cnt < pdata->channel_num && chanirq_res); 1243d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1244027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1245d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* Create DMA Channel */ 1246300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm for (i = 0; i < irq_cnt; i++) { 1247027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski err = sh_dmae_chan_probe(shdev, i, chan_irq[i], chan_flag[i]); 1248d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu if (err) 1249d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu goto chan_probe_err; 1250d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu } 1251d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1252300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm if (irq_cap) 1253300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm dev_notice(&pdev->dev, "Attempting to register %d DMA " 1254300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm "channels when a maximum of %d are supported.\n", 1255300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm pdata->channel_num, SH_DMAC_MAX_CHANNELS); 1256300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm 125720f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski pm_runtime_put(&pdev->dev); 125820f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski 1259d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu platform_set_drvdata(pdev, shdev); 1260d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_async_device_register(&shdev->common); 1261d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1262d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return err; 1263d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1264d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsuchan_probe_err: 1265d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu sh_dmae_chan_remove(shdev); 1266300e5f97d2a32196cbe03104cd6ffe2af97d9338Magnus Damm 1267927a7c9c1793def3a55d60c926d3945528e6bf1bMagnus Damm#if defined(CONFIG_CPU_SH4) || defined(CONFIG_ARCH_SHMOBILE) 1268027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski free_irq(errirq, shdev); 1269d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsueirq_err: 1270027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski#endif 1271d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsurst_err: 127231705e21f9b5a0628c043f88ff4d20488b47b8abGuennadi Liakhovetski spin_lock_irq(&sh_dmae_lock); 127303aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt list_del_rcu(&shdev->node); 127431705e21f9b5a0628c043f88ff4d20488b47b8abGuennadi Liakhovetski spin_unlock_irq(&sh_dmae_lock); 127503aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 127620f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski pm_runtime_put(&pdev->dev); 1277467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski pm_runtime_disable(&pdev->dev); 1278467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1279027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (dmars) 1280027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski iounmap(shdev->dmars); 1281027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskiemapdmars: 1282027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski iounmap(shdev->chan_reg); 128331705e21f9b5a0628c043f88ff4d20488b47b8abGuennadi Liakhovetski synchronize_rcu(); 1284027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskiemapchan: 1285d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu kfree(shdev); 1286027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskiealloc: 1287027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (dmars) 1288027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski release_mem_region(dmars->start, resource_size(dmars)); 1289027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetskiermrdmars: 1290027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski release_mem_region(chan->start, resource_size(chan)); 1291d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1292d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return err; 1293d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 1294d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1295d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic int __exit sh_dmae_remove(struct platform_device *pdev) 1296d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1297d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_device *shdev = platform_get_drvdata(pdev); 1298027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski struct resource *res; 1299027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski int errirq = platform_get_irq(pdev, 0); 1300d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1301d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu dma_async_device_unregister(&shdev->common); 1302d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1303027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (errirq > 0) 1304027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski free_irq(errirq, shdev); 1305d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 130631705e21f9b5a0628c043f88ff4d20488b47b8abGuennadi Liakhovetski spin_lock_irq(&sh_dmae_lock); 130703aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt list_del_rcu(&shdev->node); 130831705e21f9b5a0628c043f88ff4d20488b47b8abGuennadi Liakhovetski spin_unlock_irq(&sh_dmae_lock); 130903aa18f550900855c1d3d17ac83c14a3d668d344Paul Mundt 1310d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu /* channel data remove */ 1311d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu sh_dmae_chan_remove(shdev); 1312d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 131320f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski pm_runtime_disable(&pdev->dev); 131420f2a3b5d57701c54bdd59b89dd37fe775926baeGuennadi Liakhovetski 1315027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (shdev->dmars) 1316027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski iounmap(shdev->dmars); 1317027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski iounmap(shdev->chan_reg); 1318027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 131931705e21f9b5a0628c043f88ff4d20488b47b8abGuennadi Liakhovetski synchronize_rcu(); 1320d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu kfree(shdev); 1321d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1322027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1323027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (res) 1324027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski release_mem_region(res->start, resource_size(res)); 1325027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski res = platform_get_resource(pdev, IORESOURCE_MEM, 1); 1326027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski if (res) 1327027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski release_mem_region(res->start, resource_size(res)); 1328027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski 1329d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return 0; 1330d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 1331d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1332d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void sh_dmae_shutdown(struct platform_device *pdev) 1333d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1334d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu struct sh_dmae_device *shdev = platform_get_drvdata(pdev); 1335027811b9b81a6b3ae5aa20c3302897bee9dcf09eGuennadi Liakhovetski sh_dmae_ctl_stop(shdev); 1336d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 1337d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1338467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetskistatic int sh_dmae_runtime_suspend(struct device *dev) 1339467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski{ 1340467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski return 0; 1341467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski} 1342467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1343467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetskistatic int sh_dmae_runtime_resume(struct device *dev) 1344467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski{ 1345467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski struct sh_dmae_device *shdev = dev_get_drvdata(dev); 1346467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1347467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski return sh_dmae_rst(shdev); 1348467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski} 1349467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1350467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski#ifdef CONFIG_PM 1351467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetskistatic int sh_dmae_suspend(struct device *dev) 1352467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski{ 1353467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski struct sh_dmae_device *shdev = dev_get_drvdata(dev); 1354467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski int i; 1355467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1356467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski for (i = 0; i < shdev->pdata->channel_num; i++) { 1357467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski struct sh_dmae_chan *sh_chan = shdev->chan[i]; 1358467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski if (sh_chan->descs_allocated) 1359467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski sh_chan->pm_error = pm_runtime_put_sync(dev); 1360467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski } 1361467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1362467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski return 0; 1363467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski} 1364467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1365467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetskistatic int sh_dmae_resume(struct device *dev) 1366467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski{ 1367467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski struct sh_dmae_device *shdev = dev_get_drvdata(dev); 1368467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski int i; 1369467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1370467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski for (i = 0; i < shdev->pdata->channel_num; i++) { 1371467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski struct sh_dmae_chan *sh_chan = shdev->chan[i]; 1372467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski struct sh_dmae_slave *param = sh_chan->common.private; 1373467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1374467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski if (!sh_chan->descs_allocated) 1375467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski continue; 1376467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1377467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski if (!sh_chan->pm_error) 1378467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski pm_runtime_get_sync(dev); 1379467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1380467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski if (param) { 1381467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski const struct sh_dmae_slave_config *cfg = param->config; 1382467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski dmae_set_dmars(sh_chan, cfg->mid_rid); 1383467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski dmae_set_chcr(sh_chan, cfg->chcr); 1384467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski } else { 1385467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski dmae_init(sh_chan); 1386467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski } 1387467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski } 1388467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1389467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski return 0; 1390467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski} 1391467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski#else 1392467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski#define sh_dmae_suspend NULL 1393467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski#define sh_dmae_resume NULL 1394467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski#endif 1395467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1396467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetskiconst struct dev_pm_ops sh_dmae_pm = { 1397467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski .suspend = sh_dmae_suspend, 1398467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski .resume = sh_dmae_resume, 1399467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski .runtime_suspend = sh_dmae_runtime_suspend, 1400467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski .runtime_resume = sh_dmae_runtime_resume, 1401467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski}; 1402467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski 1403d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic struct platform_driver sh_dmae_driver = { 1404d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu .remove = __exit_p(sh_dmae_remove), 1405d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu .shutdown = sh_dmae_shutdown, 1406d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu .driver = { 14077a5c106a0e8fd03a806d0da77eef10b4045c43a6Guennadi Liakhovetski .owner = THIS_MODULE, 1408d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu .name = "sh-dma-engine", 1409467017b83b5bc445be5d275cf727b4f7ba3d2b2dGuennadi Liakhovetski .pm = &sh_dmae_pm, 1410d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu }, 1411d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu}; 1412d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1413d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic int __init sh_dmae_init(void) 1414d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1415661382fe190475c17d0b3a6b5f0350b4f82f5939Guennadi Liakhovetski /* Wire up NMI handling */ 1416661382fe190475c17d0b3a6b5f0350b4f82f5939Guennadi Liakhovetski int err = register_die_notifier(&sh_dmae_nmi_notifier); 1417661382fe190475c17d0b3a6b5f0350b4f82f5939Guennadi Liakhovetski if (err) 1418661382fe190475c17d0b3a6b5f0350b4f82f5939Guennadi Liakhovetski return err; 1419661382fe190475c17d0b3a6b5f0350b4f82f5939Guennadi Liakhovetski 1420d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu return platform_driver_probe(&sh_dmae_driver, sh_dmae_probe); 1421d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 1422d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsumodule_init(sh_dmae_init); 1423d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1424d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsustatic void __exit sh_dmae_exit(void) 1425d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu{ 1426d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu platform_driver_unregister(&sh_dmae_driver); 1427661382fe190475c17d0b3a6b5f0350b4f82f5939Guennadi Liakhovetski 1428661382fe190475c17d0b3a6b5f0350b4f82f5939Guennadi Liakhovetski unregister_die_notifier(&sh_dmae_nmi_notifier); 1429d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu} 1430d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsumodule_exit(sh_dmae_exit); 1431d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro Iwamatsu 1432d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro IwamatsuMODULE_AUTHOR("Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com>"); 1433d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro IwamatsuMODULE_DESCRIPTION("Renesas SH DMA Engine driver"); 1434d8902adcc1a9fd484c8cb5e575152e32192c1ff8Nobuhiro IwamatsuMODULE_LICENSE("GPL"); 1435e5843341e3ad8ff00332376cd0745026e4b5d45fGuennadi LiakhovetskiMODULE_ALIAS("platform:sh-dma-engine"); 1436