1c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech/* 2c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. 3c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * 4c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * This program is free software; you can redistribute it and/or modify it 5c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * under the terms of the GNU General Public License as published by the Free 6c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * Software Foundation; either version 2 of the License, or (at your option) 7c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * any later version. 8c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * 9c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * This program is distributed in the hope that it will be useful, but WITHOUT 10c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * more details. 13c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * 14c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * You should have received a copy of the GNU General Public License along with 15c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * this program; if not, write to the Free Software Foundation, Inc., 59 16c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * 18c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * The full GNU General Public License is included in this distribution in the 19c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * file called COPYING. 20c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech */ 21c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 22c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech/* 23c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * This code implements the DMA subsystem. It provides a HW-neutral interface 24c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * for other kernel code to use asynchronous memory copy capabilities, 25c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * if present, and allows different HW DMA drivers to register as providing 26c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * this capability. 27c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * 28c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * Due to the fact we are accelerating what is already a relatively fast 29c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * operation, the code goes to great lengths to avoid additional overhead, 30c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * such as locking. 31c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * 32c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * LOCKING: 33c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * 34aa1e6f1a385eb2b04171ec841f3b760091e4a8eeDan Williams * The subsystem keeps a global list of dma_device structs it is protected by a 35aa1e6f1a385eb2b04171ec841f3b760091e4a8eeDan Williams * mutex, dma_list_mutex. 36c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * 37f27c580c3628d79b17f38976d842a6d7f3616e2eDan Williams * A subsystem can get access to a channel by calling dmaengine_get() followed 38f27c580c3628d79b17f38976d842a6d7f3616e2eDan Williams * by dma_find_channel(), or if it has need for an exclusive channel it can call 39f27c580c3628d79b17f38976d842a6d7f3616e2eDan Williams * dma_request_channel(). Once a channel is allocated a reference is taken 40f27c580c3628d79b17f38976d842a6d7f3616e2eDan Williams * against its corresponding driver to disable removal. 41f27c580c3628d79b17f38976d842a6d7f3616e2eDan Williams * 42c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * Each device has a channels list, which runs unlocked but is never modified 43c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * once the device is registered, it's just setup by the driver. 44c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * 45f27c580c3628d79b17f38976d842a6d7f3616e2eDan Williams * See Documentation/dmaengine.txt for more details 46c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech */ 47c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 48b7f080cfe223b3b7424872639d153695615a9255Alexey Dobriyan#include <linux/dma-mapping.h> 49c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech#include <linux/init.h> 50c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech#include <linux/module.h> 517405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams#include <linux/mm.h> 52c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech#include <linux/device.h> 53c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech#include <linux/dmaengine.h> 54c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech#include <linux/hardirq.h> 55c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech#include <linux/spinlock.h> 56c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech#include <linux/percpu.h> 57c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech#include <linux/rcupdate.h> 58c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech#include <linux/mutex.h> 597405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams#include <linux/jiffies.h> 602ba05622b8b143b0c95968ba59bddfbd6d2f2559Dan Williams#include <linux/rculist.h> 61864498aaa9fef69ee166da023d12413a7776342dDan Williams#include <linux/idr.h> 625a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 63c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 64c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leechstatic DEFINE_MUTEX(dma_list_mutex); 6521ef4b8b7a7d59a995bf44382de38c95587767d4Axel Linstatic DEFINE_IDR(dma_idr); 66c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leechstatic LIST_HEAD(dma_device_list); 676f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williamsstatic long dmaengine_ref_count; 68c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 69c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech/* --- sysfs implementation --- */ 70c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 7141d5e59c1299f27983977bcfe3b360600996051cDan Williams/** 7241d5e59c1299f27983977bcfe3b360600996051cDan Williams * dev_to_dma_chan - convert a device pointer to the its sysfs container object 7341d5e59c1299f27983977bcfe3b360600996051cDan Williams * @dev - device node 7441d5e59c1299f27983977bcfe3b360600996051cDan Williams * 7541d5e59c1299f27983977bcfe3b360600996051cDan Williams * Must be called under dma_list_mutex 7641d5e59c1299f27983977bcfe3b360600996051cDan Williams */ 7741d5e59c1299f27983977bcfe3b360600996051cDan Williamsstatic struct dma_chan *dev_to_dma_chan(struct device *dev) 7841d5e59c1299f27983977bcfe3b360600996051cDan Williams{ 7941d5e59c1299f27983977bcfe3b360600996051cDan Williams struct dma_chan_dev *chan_dev; 8041d5e59c1299f27983977bcfe3b360600996051cDan Williams 8141d5e59c1299f27983977bcfe3b360600996051cDan Williams chan_dev = container_of(dev, typeof(*chan_dev), device); 8241d5e59c1299f27983977bcfe3b360600996051cDan Williams return chan_dev->chan; 8341d5e59c1299f27983977bcfe3b360600996051cDan Williams} 8441d5e59c1299f27983977bcfe3b360600996051cDan Williams 85891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jonesstatic ssize_t show_memcpy_count(struct device *dev, struct device_attribute *attr, char *buf) 86c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 8741d5e59c1299f27983977bcfe3b360600996051cDan Williams struct dma_chan *chan; 88c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech unsigned long count = 0; 89c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech int i; 9041d5e59c1299f27983977bcfe3b360600996051cDan Williams int err; 91c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 9241d5e59c1299f27983977bcfe3b360600996051cDan Williams mutex_lock(&dma_list_mutex); 9341d5e59c1299f27983977bcfe3b360600996051cDan Williams chan = dev_to_dma_chan(dev); 9441d5e59c1299f27983977bcfe3b360600996051cDan Williams if (chan) { 9541d5e59c1299f27983977bcfe3b360600996051cDan Williams for_each_possible_cpu(i) 9641d5e59c1299f27983977bcfe3b360600996051cDan Williams count += per_cpu_ptr(chan->local, i)->memcpy_count; 9741d5e59c1299f27983977bcfe3b360600996051cDan Williams err = sprintf(buf, "%lu\n", count); 9841d5e59c1299f27983977bcfe3b360600996051cDan Williams } else 9941d5e59c1299f27983977bcfe3b360600996051cDan Williams err = -ENODEV; 10041d5e59c1299f27983977bcfe3b360600996051cDan Williams mutex_unlock(&dma_list_mutex); 101c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 10241d5e59c1299f27983977bcfe3b360600996051cDan Williams return err; 103c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 104c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 105891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jonesstatic ssize_t show_bytes_transferred(struct device *dev, struct device_attribute *attr, 106891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jones char *buf) 107c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 10841d5e59c1299f27983977bcfe3b360600996051cDan Williams struct dma_chan *chan; 109c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech unsigned long count = 0; 110c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech int i; 11141d5e59c1299f27983977bcfe3b360600996051cDan Williams int err; 112c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 11341d5e59c1299f27983977bcfe3b360600996051cDan Williams mutex_lock(&dma_list_mutex); 11441d5e59c1299f27983977bcfe3b360600996051cDan Williams chan = dev_to_dma_chan(dev); 11541d5e59c1299f27983977bcfe3b360600996051cDan Williams if (chan) { 11641d5e59c1299f27983977bcfe3b360600996051cDan Williams for_each_possible_cpu(i) 11741d5e59c1299f27983977bcfe3b360600996051cDan Williams count += per_cpu_ptr(chan->local, i)->bytes_transferred; 11841d5e59c1299f27983977bcfe3b360600996051cDan Williams err = sprintf(buf, "%lu\n", count); 11941d5e59c1299f27983977bcfe3b360600996051cDan Williams } else 12041d5e59c1299f27983977bcfe3b360600996051cDan Williams err = -ENODEV; 12141d5e59c1299f27983977bcfe3b360600996051cDan Williams mutex_unlock(&dma_list_mutex); 122c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 12341d5e59c1299f27983977bcfe3b360600996051cDan Williams return err; 124c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 125c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 126891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jonesstatic ssize_t show_in_use(struct device *dev, struct device_attribute *attr, char *buf) 127c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 12841d5e59c1299f27983977bcfe3b360600996051cDan Williams struct dma_chan *chan; 12941d5e59c1299f27983977bcfe3b360600996051cDan Williams int err; 130c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 13141d5e59c1299f27983977bcfe3b360600996051cDan Williams mutex_lock(&dma_list_mutex); 13241d5e59c1299f27983977bcfe3b360600996051cDan Williams chan = dev_to_dma_chan(dev); 13341d5e59c1299f27983977bcfe3b360600996051cDan Williams if (chan) 13441d5e59c1299f27983977bcfe3b360600996051cDan Williams err = sprintf(buf, "%d\n", chan->client_count); 13541d5e59c1299f27983977bcfe3b360600996051cDan Williams else 13641d5e59c1299f27983977bcfe3b360600996051cDan Williams err = -ENODEV; 13741d5e59c1299f27983977bcfe3b360600996051cDan Williams mutex_unlock(&dma_list_mutex); 13841d5e59c1299f27983977bcfe3b360600996051cDan Williams 13941d5e59c1299f27983977bcfe3b360600996051cDan Williams return err; 140c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 141c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 142891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jonesstatic struct device_attribute dma_attrs[] = { 143c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech __ATTR(memcpy_count, S_IRUGO, show_memcpy_count, NULL), 144c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech __ATTR(bytes_transferred, S_IRUGO, show_bytes_transferred, NULL), 145c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech __ATTR(in_use, S_IRUGO, show_in_use, NULL), 146c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech __ATTR_NULL 147c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech}; 148c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 14941d5e59c1299f27983977bcfe3b360600996051cDan Williamsstatic void chan_dev_release(struct device *dev) 15041d5e59c1299f27983977bcfe3b360600996051cDan Williams{ 15141d5e59c1299f27983977bcfe3b360600996051cDan Williams struct dma_chan_dev *chan_dev; 15241d5e59c1299f27983977bcfe3b360600996051cDan Williams 15341d5e59c1299f27983977bcfe3b360600996051cDan Williams chan_dev = container_of(dev, typeof(*chan_dev), device); 154864498aaa9fef69ee166da023d12413a7776342dDan Williams if (atomic_dec_and_test(chan_dev->idr_ref)) { 155864498aaa9fef69ee166da023d12413a7776342dDan Williams mutex_lock(&dma_list_mutex); 156864498aaa9fef69ee166da023d12413a7776342dDan Williams idr_remove(&dma_idr, chan_dev->dev_id); 157864498aaa9fef69ee166da023d12413a7776342dDan Williams mutex_unlock(&dma_list_mutex); 158864498aaa9fef69ee166da023d12413a7776342dDan Williams kfree(chan_dev->idr_ref); 159864498aaa9fef69ee166da023d12413a7776342dDan Williams } 16041d5e59c1299f27983977bcfe3b360600996051cDan Williams kfree(chan_dev); 16141d5e59c1299f27983977bcfe3b360600996051cDan Williams} 16241d5e59c1299f27983977bcfe3b360600996051cDan Williams 163c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leechstatic struct class dma_devclass = { 164891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jones .name = "dma", 165891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jones .dev_attrs = dma_attrs, 16641d5e59c1299f27983977bcfe3b360600996051cDan Williams .dev_release = chan_dev_release, 167c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech}; 168c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 169c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech/* --- client and device registration --- */ 170c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 17159b5ec21446b9239d706ab237fb261d525b75e81Dan Williams#define dma_device_satisfies_mask(device, mask) \ 17259b5ec21446b9239d706ab237fb261d525b75e81Dan Williams __dma_device_satisfies_mask((device), &(mask)) 173d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williamsstatic int 17459b5ec21446b9239d706ab237fb261d525b75e81Dan Williams__dma_device_satisfies_mask(struct dma_device *device, dma_cap_mask_t *want) 175d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams{ 176d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams dma_cap_mask_t has; 177d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams 17859b5ec21446b9239d706ab237fb261d525b75e81Dan Williams bitmap_and(has.bits, want->bits, device->cap_mask.bits, 179d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams DMA_TX_TYPE_END); 180d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END); 181d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams} 182d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams 1836f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williamsstatic struct module *dma_chan_to_owner(struct dma_chan *chan) 1846f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams{ 1856f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams return chan->device->dev->driver->owner; 1866f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams} 1876f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams 1886f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams/** 1896f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams * balance_ref_count - catch up the channel reference count 1906f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams * @chan - channel to balance ->client_count versus dmaengine_ref_count 1916f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams * 1926f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams * balance_ref_count must be called under dma_list_mutex 1936f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams */ 1946f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williamsstatic void balance_ref_count(struct dma_chan *chan) 1956f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams{ 1966f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams struct module *owner = dma_chan_to_owner(chan); 1976f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams 1986f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams while (chan->client_count < dmaengine_ref_count) { 1996f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams __module_get(owner); 2006f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams chan->client_count++; 2016f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams } 2026f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams} 2036f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams 2046f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams/** 2056f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams * dma_chan_get - try to grab a dma channel's parent driver module 2066f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams * @chan - channel to grab 2076f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams * 2086f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams * Must be called under dma_list_mutex 2096f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams */ 2106f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williamsstatic int dma_chan_get(struct dma_chan *chan) 2116f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams{ 2126f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams int err = -ENODEV; 2136f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams struct module *owner = dma_chan_to_owner(chan); 2146f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams 2156f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams if (chan->client_count) { 2166f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams __module_get(owner); 2176f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams err = 0; 2186f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams } else if (try_module_get(owner)) 2196f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams err = 0; 2206f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams 2216f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams if (err == 0) 2226f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams chan->client_count++; 2236f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams 2246f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams /* allocate upon first client reference */ 2256f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams if (chan->client_count == 1 && err == 0) { 226aa1e6f1a385eb2b04171ec841f3b760091e4a8eeDan Williams int desc_cnt = chan->device->device_alloc_chan_resources(chan); 2276f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams 2286f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams if (desc_cnt < 0) { 2296f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams err = desc_cnt; 2306f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams chan->client_count = 0; 2316f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams module_put(owner); 23259b5ec21446b9239d706ab237fb261d525b75e81Dan Williams } else if (!dma_has_cap(DMA_PRIVATE, chan->device->cap_mask)) 2336f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams balance_ref_count(chan); 2346f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams } 2356f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams 2366f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams return err; 2376f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams} 2386f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams 2396f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams/** 2406f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams * dma_chan_put - drop a reference to a dma channel's parent driver module 2416f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams * @chan - channel to release 2426f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams * 2436f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams * Must be called under dma_list_mutex 2446f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams */ 2456f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williamsstatic void dma_chan_put(struct dma_chan *chan) 2466f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams{ 2476f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams if (!chan->client_count) 2486f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams return; /* this channel failed alloc_chan_resources */ 2496f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams chan->client_count--; 2506f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams module_put(dma_chan_to_owner(chan)); 2516f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams if (chan->client_count == 0) 2526f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams chan->device->device_free_chan_resources(chan); 2536f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams} 2546f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams 2557405f74badf46b5d023c5d2b670b4471525f6c91Dan Williamsenum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie) 2567405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams{ 2577405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams enum dma_status status; 2587405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000); 2597405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 2607405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams dma_async_issue_pending(chan); 2617405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams do { 2627405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams status = dma_async_is_tx_complete(chan, cookie, NULL, NULL); 2637405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams if (time_after_eq(jiffies, dma_sync_wait_timeout)) { 2647405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams printk(KERN_ERR "dma_sync_wait_timeout!\n"); 2657405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams return DMA_ERROR; 2667405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams } 2677405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams } while (status == DMA_IN_PROGRESS); 2687405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 2697405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams return status; 2707405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams} 2717405f74badf46b5d023c5d2b670b4471525f6c91Dan WilliamsEXPORT_SYMBOL(dma_sync_wait); 2727405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 273c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech/** 274bec085134e446577a983f17f57d642a88d1af53bDan Williams * dma_cap_mask_all - enable iteration over all operation types 275bec085134e446577a983f17f57d642a88d1af53bDan Williams */ 276bec085134e446577a983f17f57d642a88d1af53bDan Williamsstatic dma_cap_mask_t dma_cap_mask_all; 277bec085134e446577a983f17f57d642a88d1af53bDan Williams 278bec085134e446577a983f17f57d642a88d1af53bDan Williams/** 279bec085134e446577a983f17f57d642a88d1af53bDan Williams * dma_chan_tbl_ent - tracks channel allocations per core/operation 280bec085134e446577a983f17f57d642a88d1af53bDan Williams * @chan - associated channel for this entry 281bec085134e446577a983f17f57d642a88d1af53bDan Williams */ 282bec085134e446577a983f17f57d642a88d1af53bDan Williamsstruct dma_chan_tbl_ent { 283bec085134e446577a983f17f57d642a88d1af53bDan Williams struct dma_chan *chan; 284bec085134e446577a983f17f57d642a88d1af53bDan Williams}; 285bec085134e446577a983f17f57d642a88d1af53bDan Williams 286bec085134e446577a983f17f57d642a88d1af53bDan Williams/** 287bec085134e446577a983f17f57d642a88d1af53bDan Williams * channel_table - percpu lookup table for memory-to-memory offload providers 288bec085134e446577a983f17f57d642a88d1af53bDan Williams */ 289a29d8b8e2d811a24bbe49215a0f0c536b72ebc18Tejun Heostatic struct dma_chan_tbl_ent __percpu *channel_table[DMA_TX_TYPE_END]; 290bec085134e446577a983f17f57d642a88d1af53bDan Williams 291bec085134e446577a983f17f57d642a88d1af53bDan Williamsstatic int __init dma_channel_table_init(void) 292bec085134e446577a983f17f57d642a88d1af53bDan Williams{ 293bec085134e446577a983f17f57d642a88d1af53bDan Williams enum dma_transaction_type cap; 294bec085134e446577a983f17f57d642a88d1af53bDan Williams int err = 0; 295bec085134e446577a983f17f57d642a88d1af53bDan Williams 296bec085134e446577a983f17f57d642a88d1af53bDan Williams bitmap_fill(dma_cap_mask_all.bits, DMA_TX_TYPE_END); 297bec085134e446577a983f17f57d642a88d1af53bDan Williams 29859b5ec21446b9239d706ab237fb261d525b75e81Dan Williams /* 'interrupt', 'private', and 'slave' are channel capabilities, 29959b5ec21446b9239d706ab237fb261d525b75e81Dan Williams * but are not associated with an operation so they do not need 30059b5ec21446b9239d706ab237fb261d525b75e81Dan Williams * an entry in the channel_table 301bec085134e446577a983f17f57d642a88d1af53bDan Williams */ 302bec085134e446577a983f17f57d642a88d1af53bDan Williams clear_bit(DMA_INTERRUPT, dma_cap_mask_all.bits); 30359b5ec21446b9239d706ab237fb261d525b75e81Dan Williams clear_bit(DMA_PRIVATE, dma_cap_mask_all.bits); 304bec085134e446577a983f17f57d642a88d1af53bDan Williams clear_bit(DMA_SLAVE, dma_cap_mask_all.bits); 305bec085134e446577a983f17f57d642a88d1af53bDan Williams 306bec085134e446577a983f17f57d642a88d1af53bDan Williams for_each_dma_cap_mask(cap, dma_cap_mask_all) { 307bec085134e446577a983f17f57d642a88d1af53bDan Williams channel_table[cap] = alloc_percpu(struct dma_chan_tbl_ent); 308bec085134e446577a983f17f57d642a88d1af53bDan Williams if (!channel_table[cap]) { 309bec085134e446577a983f17f57d642a88d1af53bDan Williams err = -ENOMEM; 310bec085134e446577a983f17f57d642a88d1af53bDan Williams break; 311bec085134e446577a983f17f57d642a88d1af53bDan Williams } 312bec085134e446577a983f17f57d642a88d1af53bDan Williams } 313bec085134e446577a983f17f57d642a88d1af53bDan Williams 314bec085134e446577a983f17f57d642a88d1af53bDan Williams if (err) { 315bec085134e446577a983f17f57d642a88d1af53bDan Williams pr_err("dmaengine: initialization failure\n"); 316bec085134e446577a983f17f57d642a88d1af53bDan Williams for_each_dma_cap_mask(cap, dma_cap_mask_all) 317bec085134e446577a983f17f57d642a88d1af53bDan Williams if (channel_table[cap]) 318bec085134e446577a983f17f57d642a88d1af53bDan Williams free_percpu(channel_table[cap]); 319bec085134e446577a983f17f57d642a88d1af53bDan Williams } 320bec085134e446577a983f17f57d642a88d1af53bDan Williams 321bec085134e446577a983f17f57d642a88d1af53bDan Williams return err; 322bec085134e446577a983f17f57d642a88d1af53bDan Williams} 323652afc27b26859a0ea5f6db681d80b83d2c43cf8Dan Williamsarch_initcall(dma_channel_table_init); 324bec085134e446577a983f17f57d642a88d1af53bDan Williams 325bec085134e446577a983f17f57d642a88d1af53bDan Williams/** 326bec085134e446577a983f17f57d642a88d1af53bDan Williams * dma_find_channel - find a channel to carry out the operation 327bec085134e446577a983f17f57d642a88d1af53bDan Williams * @tx_type: transaction type 328bec085134e446577a983f17f57d642a88d1af53bDan Williams */ 329bec085134e446577a983f17f57d642a88d1af53bDan Williamsstruct dma_chan *dma_find_channel(enum dma_transaction_type tx_type) 330bec085134e446577a983f17f57d642a88d1af53bDan Williams{ 331e7dcaa4755e35d7540bf19f316f8798357c53fa0Christoph Lameter return this_cpu_read(channel_table[tx_type]->chan); 332bec085134e446577a983f17f57d642a88d1af53bDan Williams} 333bec085134e446577a983f17f57d642a88d1af53bDan WilliamsEXPORT_SYMBOL(dma_find_channel); 334bec085134e446577a983f17f57d642a88d1af53bDan Williams 335bec085134e446577a983f17f57d642a88d1af53bDan Williams/** 3362ba05622b8b143b0c95968ba59bddfbd6d2f2559Dan Williams * dma_issue_pending_all - flush all pending operations across all channels 3372ba05622b8b143b0c95968ba59bddfbd6d2f2559Dan Williams */ 3382ba05622b8b143b0c95968ba59bddfbd6d2f2559Dan Williamsvoid dma_issue_pending_all(void) 3392ba05622b8b143b0c95968ba59bddfbd6d2f2559Dan Williams{ 3402ba05622b8b143b0c95968ba59bddfbd6d2f2559Dan Williams struct dma_device *device; 3412ba05622b8b143b0c95968ba59bddfbd6d2f2559Dan Williams struct dma_chan *chan; 3422ba05622b8b143b0c95968ba59bddfbd6d2f2559Dan Williams 3432ba05622b8b143b0c95968ba59bddfbd6d2f2559Dan Williams rcu_read_lock(); 34459b5ec21446b9239d706ab237fb261d525b75e81Dan Williams list_for_each_entry_rcu(device, &dma_device_list, global_node) { 34559b5ec21446b9239d706ab237fb261d525b75e81Dan Williams if (dma_has_cap(DMA_PRIVATE, device->cap_mask)) 34659b5ec21446b9239d706ab237fb261d525b75e81Dan Williams continue; 3472ba05622b8b143b0c95968ba59bddfbd6d2f2559Dan Williams list_for_each_entry(chan, &device->channels, device_node) 3482ba05622b8b143b0c95968ba59bddfbd6d2f2559Dan Williams if (chan->client_count) 3492ba05622b8b143b0c95968ba59bddfbd6d2f2559Dan Williams device->device_issue_pending(chan); 35059b5ec21446b9239d706ab237fb261d525b75e81Dan Williams } 3512ba05622b8b143b0c95968ba59bddfbd6d2f2559Dan Williams rcu_read_unlock(); 3522ba05622b8b143b0c95968ba59bddfbd6d2f2559Dan Williams} 3532ba05622b8b143b0c95968ba59bddfbd6d2f2559Dan WilliamsEXPORT_SYMBOL(dma_issue_pending_all); 3542ba05622b8b143b0c95968ba59bddfbd6d2f2559Dan Williams 3552ba05622b8b143b0c95968ba59bddfbd6d2f2559Dan Williams/** 356bec085134e446577a983f17f57d642a88d1af53bDan Williams * nth_chan - returns the nth channel of the given capability 357bec085134e446577a983f17f57d642a88d1af53bDan Williams * @cap: capability to match 358bec085134e446577a983f17f57d642a88d1af53bDan Williams * @n: nth channel desired 359bec085134e446577a983f17f57d642a88d1af53bDan Williams * 360bec085134e446577a983f17f57d642a88d1af53bDan Williams * Defaults to returning the channel with the desired capability and the 361bec085134e446577a983f17f57d642a88d1af53bDan Williams * lowest reference count when 'n' cannot be satisfied. Must be called 362bec085134e446577a983f17f57d642a88d1af53bDan Williams * under dma_list_mutex. 363bec085134e446577a983f17f57d642a88d1af53bDan Williams */ 364bec085134e446577a983f17f57d642a88d1af53bDan Williamsstatic struct dma_chan *nth_chan(enum dma_transaction_type cap, int n) 365bec085134e446577a983f17f57d642a88d1af53bDan Williams{ 366bec085134e446577a983f17f57d642a88d1af53bDan Williams struct dma_device *device; 367bec085134e446577a983f17f57d642a88d1af53bDan Williams struct dma_chan *chan; 368bec085134e446577a983f17f57d642a88d1af53bDan Williams struct dma_chan *ret = NULL; 369bec085134e446577a983f17f57d642a88d1af53bDan Williams struct dma_chan *min = NULL; 370bec085134e446577a983f17f57d642a88d1af53bDan Williams 371bec085134e446577a983f17f57d642a88d1af53bDan Williams list_for_each_entry(device, &dma_device_list, global_node) { 37259b5ec21446b9239d706ab237fb261d525b75e81Dan Williams if (!dma_has_cap(cap, device->cap_mask) || 37359b5ec21446b9239d706ab237fb261d525b75e81Dan Williams dma_has_cap(DMA_PRIVATE, device->cap_mask)) 374bec085134e446577a983f17f57d642a88d1af53bDan Williams continue; 375bec085134e446577a983f17f57d642a88d1af53bDan Williams list_for_each_entry(chan, &device->channels, device_node) { 376bec085134e446577a983f17f57d642a88d1af53bDan Williams if (!chan->client_count) 377bec085134e446577a983f17f57d642a88d1af53bDan Williams continue; 378bec085134e446577a983f17f57d642a88d1af53bDan Williams if (!min) 379bec085134e446577a983f17f57d642a88d1af53bDan Williams min = chan; 380bec085134e446577a983f17f57d642a88d1af53bDan Williams else if (chan->table_count < min->table_count) 381bec085134e446577a983f17f57d642a88d1af53bDan Williams min = chan; 382bec085134e446577a983f17f57d642a88d1af53bDan Williams 383bec085134e446577a983f17f57d642a88d1af53bDan Williams if (n-- == 0) { 384bec085134e446577a983f17f57d642a88d1af53bDan Williams ret = chan; 385bec085134e446577a983f17f57d642a88d1af53bDan Williams break; /* done */ 386bec085134e446577a983f17f57d642a88d1af53bDan Williams } 387bec085134e446577a983f17f57d642a88d1af53bDan Williams } 388bec085134e446577a983f17f57d642a88d1af53bDan Williams if (ret) 389bec085134e446577a983f17f57d642a88d1af53bDan Williams break; /* done */ 390bec085134e446577a983f17f57d642a88d1af53bDan Williams } 391bec085134e446577a983f17f57d642a88d1af53bDan Williams 392bec085134e446577a983f17f57d642a88d1af53bDan Williams if (!ret) 393bec085134e446577a983f17f57d642a88d1af53bDan Williams ret = min; 394bec085134e446577a983f17f57d642a88d1af53bDan Williams 395bec085134e446577a983f17f57d642a88d1af53bDan Williams if (ret) 396bec085134e446577a983f17f57d642a88d1af53bDan Williams ret->table_count++; 397bec085134e446577a983f17f57d642a88d1af53bDan Williams 398bec085134e446577a983f17f57d642a88d1af53bDan Williams return ret; 399bec085134e446577a983f17f57d642a88d1af53bDan Williams} 400bec085134e446577a983f17f57d642a88d1af53bDan Williams 401bec085134e446577a983f17f57d642a88d1af53bDan Williams/** 402bec085134e446577a983f17f57d642a88d1af53bDan Williams * dma_channel_rebalance - redistribute the available channels 403bec085134e446577a983f17f57d642a88d1af53bDan Williams * 404bec085134e446577a983f17f57d642a88d1af53bDan Williams * Optimize for cpu isolation (each cpu gets a dedicated channel for an 405bec085134e446577a983f17f57d642a88d1af53bDan Williams * operation type) in the SMP case, and operation isolation (avoid 406bec085134e446577a983f17f57d642a88d1af53bDan Williams * multi-tasking channels) in the non-SMP case. Must be called under 407bec085134e446577a983f17f57d642a88d1af53bDan Williams * dma_list_mutex. 408bec085134e446577a983f17f57d642a88d1af53bDan Williams */ 409bec085134e446577a983f17f57d642a88d1af53bDan Williamsstatic void dma_channel_rebalance(void) 410bec085134e446577a983f17f57d642a88d1af53bDan Williams{ 411bec085134e446577a983f17f57d642a88d1af53bDan Williams struct dma_chan *chan; 412bec085134e446577a983f17f57d642a88d1af53bDan Williams struct dma_device *device; 413bec085134e446577a983f17f57d642a88d1af53bDan Williams int cpu; 414bec085134e446577a983f17f57d642a88d1af53bDan Williams int cap; 415bec085134e446577a983f17f57d642a88d1af53bDan Williams int n; 416bec085134e446577a983f17f57d642a88d1af53bDan Williams 417bec085134e446577a983f17f57d642a88d1af53bDan Williams /* undo the last distribution */ 418bec085134e446577a983f17f57d642a88d1af53bDan Williams for_each_dma_cap_mask(cap, dma_cap_mask_all) 419bec085134e446577a983f17f57d642a88d1af53bDan Williams for_each_possible_cpu(cpu) 420bec085134e446577a983f17f57d642a88d1af53bDan Williams per_cpu_ptr(channel_table[cap], cpu)->chan = NULL; 421bec085134e446577a983f17f57d642a88d1af53bDan Williams 42259b5ec21446b9239d706ab237fb261d525b75e81Dan Williams list_for_each_entry(device, &dma_device_list, global_node) { 42359b5ec21446b9239d706ab237fb261d525b75e81Dan Williams if (dma_has_cap(DMA_PRIVATE, device->cap_mask)) 42459b5ec21446b9239d706ab237fb261d525b75e81Dan Williams continue; 425bec085134e446577a983f17f57d642a88d1af53bDan Williams list_for_each_entry(chan, &device->channels, device_node) 426bec085134e446577a983f17f57d642a88d1af53bDan Williams chan->table_count = 0; 42759b5ec21446b9239d706ab237fb261d525b75e81Dan Williams } 428bec085134e446577a983f17f57d642a88d1af53bDan Williams 429bec085134e446577a983f17f57d642a88d1af53bDan Williams /* don't populate the channel_table if no clients are available */ 430bec085134e446577a983f17f57d642a88d1af53bDan Williams if (!dmaengine_ref_count) 431bec085134e446577a983f17f57d642a88d1af53bDan Williams return; 432bec085134e446577a983f17f57d642a88d1af53bDan Williams 433bec085134e446577a983f17f57d642a88d1af53bDan Williams /* redistribute available channels */ 434bec085134e446577a983f17f57d642a88d1af53bDan Williams n = 0; 435bec085134e446577a983f17f57d642a88d1af53bDan Williams for_each_dma_cap_mask(cap, dma_cap_mask_all) 436bec085134e446577a983f17f57d642a88d1af53bDan Williams for_each_online_cpu(cpu) { 437bec085134e446577a983f17f57d642a88d1af53bDan Williams if (num_possible_cpus() > 1) 438bec085134e446577a983f17f57d642a88d1af53bDan Williams chan = nth_chan(cap, n++); 439bec085134e446577a983f17f57d642a88d1af53bDan Williams else 440bec085134e446577a983f17f57d642a88d1af53bDan Williams chan = nth_chan(cap, -1); 441bec085134e446577a983f17f57d642a88d1af53bDan Williams 442bec085134e446577a983f17f57d642a88d1af53bDan Williams per_cpu_ptr(channel_table[cap], cpu)->chan = chan; 443bec085134e446577a983f17f57d642a88d1af53bDan Williams } 444bec085134e446577a983f17f57d642a88d1af53bDan Williams} 445bec085134e446577a983f17f57d642a88d1af53bDan Williams 446e2346677af86150c6083974585c131e8a2c3ddccDan Williamsstatic struct dma_chan *private_candidate(dma_cap_mask_t *mask, struct dma_device *dev, 447e2346677af86150c6083974585c131e8a2c3ddccDan Williams dma_filter_fn fn, void *fn_param) 44859b5ec21446b9239d706ab237fb261d525b75e81Dan Williams{ 44959b5ec21446b9239d706ab237fb261d525b75e81Dan Williams struct dma_chan *chan; 45059b5ec21446b9239d706ab237fb261d525b75e81Dan Williams 45159b5ec21446b9239d706ab237fb261d525b75e81Dan Williams if (!__dma_device_satisfies_mask(dev, mask)) { 45259b5ec21446b9239d706ab237fb261d525b75e81Dan Williams pr_debug("%s: wrong capabilities\n", __func__); 45359b5ec21446b9239d706ab237fb261d525b75e81Dan Williams return NULL; 45459b5ec21446b9239d706ab237fb261d525b75e81Dan Williams } 45559b5ec21446b9239d706ab237fb261d525b75e81Dan Williams /* devices with multiple channels need special handling as we need to 45659b5ec21446b9239d706ab237fb261d525b75e81Dan Williams * ensure that all channels are either private or public. 45759b5ec21446b9239d706ab237fb261d525b75e81Dan Williams */ 45859b5ec21446b9239d706ab237fb261d525b75e81Dan Williams if (dev->chancnt > 1 && !dma_has_cap(DMA_PRIVATE, dev->cap_mask)) 45959b5ec21446b9239d706ab237fb261d525b75e81Dan Williams list_for_each_entry(chan, &dev->channels, device_node) { 46059b5ec21446b9239d706ab237fb261d525b75e81Dan Williams /* some channels are already publicly allocated */ 46159b5ec21446b9239d706ab237fb261d525b75e81Dan Williams if (chan->client_count) 46259b5ec21446b9239d706ab237fb261d525b75e81Dan Williams return NULL; 46359b5ec21446b9239d706ab237fb261d525b75e81Dan Williams } 46459b5ec21446b9239d706ab237fb261d525b75e81Dan Williams 46559b5ec21446b9239d706ab237fb261d525b75e81Dan Williams list_for_each_entry(chan, &dev->channels, device_node) { 46659b5ec21446b9239d706ab237fb261d525b75e81Dan Williams if (chan->client_count) { 46759b5ec21446b9239d706ab237fb261d525b75e81Dan Williams pr_debug("%s: %s busy\n", 46841d5e59c1299f27983977bcfe3b360600996051cDan Williams __func__, dma_chan_name(chan)); 46959b5ec21446b9239d706ab237fb261d525b75e81Dan Williams continue; 47059b5ec21446b9239d706ab237fb261d525b75e81Dan Williams } 471e2346677af86150c6083974585c131e8a2c3ddccDan Williams if (fn && !fn(chan, fn_param)) { 472e2346677af86150c6083974585c131e8a2c3ddccDan Williams pr_debug("%s: %s filter said false\n", 473e2346677af86150c6083974585c131e8a2c3ddccDan Williams __func__, dma_chan_name(chan)); 474e2346677af86150c6083974585c131e8a2c3ddccDan Williams continue; 475e2346677af86150c6083974585c131e8a2c3ddccDan Williams } 476e2346677af86150c6083974585c131e8a2c3ddccDan Williams return chan; 47759b5ec21446b9239d706ab237fb261d525b75e81Dan Williams } 47859b5ec21446b9239d706ab237fb261d525b75e81Dan Williams 479e2346677af86150c6083974585c131e8a2c3ddccDan Williams return NULL; 48059b5ec21446b9239d706ab237fb261d525b75e81Dan Williams} 48159b5ec21446b9239d706ab237fb261d525b75e81Dan Williams 48259b5ec21446b9239d706ab237fb261d525b75e81Dan Williams/** 48359b5ec21446b9239d706ab237fb261d525b75e81Dan Williams * dma_request_channel - try to allocate an exclusive channel 48459b5ec21446b9239d706ab237fb261d525b75e81Dan Williams * @mask: capabilities that the channel must satisfy 48559b5ec21446b9239d706ab237fb261d525b75e81Dan Williams * @fn: optional callback to disposition available channels 48659b5ec21446b9239d706ab237fb261d525b75e81Dan Williams * @fn_param: opaque parameter to pass to dma_filter_fn 48759b5ec21446b9239d706ab237fb261d525b75e81Dan Williams */ 48859b5ec21446b9239d706ab237fb261d525b75e81Dan Williamsstruct dma_chan *__dma_request_channel(dma_cap_mask_t *mask, dma_filter_fn fn, void *fn_param) 48959b5ec21446b9239d706ab237fb261d525b75e81Dan Williams{ 49059b5ec21446b9239d706ab237fb261d525b75e81Dan Williams struct dma_device *device, *_d; 49159b5ec21446b9239d706ab237fb261d525b75e81Dan Williams struct dma_chan *chan = NULL; 49259b5ec21446b9239d706ab237fb261d525b75e81Dan Williams int err; 49359b5ec21446b9239d706ab237fb261d525b75e81Dan Williams 49459b5ec21446b9239d706ab237fb261d525b75e81Dan Williams /* Find a channel */ 49559b5ec21446b9239d706ab237fb261d525b75e81Dan Williams mutex_lock(&dma_list_mutex); 49659b5ec21446b9239d706ab237fb261d525b75e81Dan Williams list_for_each_entry_safe(device, _d, &dma_device_list, global_node) { 497e2346677af86150c6083974585c131e8a2c3ddccDan Williams chan = private_candidate(mask, device, fn, fn_param); 498e2346677af86150c6083974585c131e8a2c3ddccDan Williams if (chan) { 49959b5ec21446b9239d706ab237fb261d525b75e81Dan Williams /* Found a suitable channel, try to grab, prep, and 50059b5ec21446b9239d706ab237fb261d525b75e81Dan Williams * return it. We first set DMA_PRIVATE to disable 50159b5ec21446b9239d706ab237fb261d525b75e81Dan Williams * balance_ref_count as this channel will not be 50259b5ec21446b9239d706ab237fb261d525b75e81Dan Williams * published in the general-purpose allocator 50359b5ec21446b9239d706ab237fb261d525b75e81Dan Williams */ 50459b5ec21446b9239d706ab237fb261d525b75e81Dan Williams dma_cap_set(DMA_PRIVATE, device->cap_mask); 5050f571515c332e00b3515dbe0859ceaa30ab66e00Atsushi Nemoto device->privatecnt++; 50659b5ec21446b9239d706ab237fb261d525b75e81Dan Williams err = dma_chan_get(chan); 50759b5ec21446b9239d706ab237fb261d525b75e81Dan Williams 50859b5ec21446b9239d706ab237fb261d525b75e81Dan Williams if (err == -ENODEV) { 50959b5ec21446b9239d706ab237fb261d525b75e81Dan Williams pr_debug("%s: %s module removed\n", __func__, 51041d5e59c1299f27983977bcfe3b360600996051cDan Williams dma_chan_name(chan)); 51159b5ec21446b9239d706ab237fb261d525b75e81Dan Williams list_del_rcu(&device->global_node); 51259b5ec21446b9239d706ab237fb261d525b75e81Dan Williams } else if (err) 513a03a202e95fdaa3ff52ccfc2594ec531e5917816Guennadi Liakhovetski pr_debug("dmaengine: failed to get %s: (%d)\n", 514a03a202e95fdaa3ff52ccfc2594ec531e5917816Guennadi Liakhovetski dma_chan_name(chan), err); 51559b5ec21446b9239d706ab237fb261d525b75e81Dan Williams else 51659b5ec21446b9239d706ab237fb261d525b75e81Dan Williams break; 5170f571515c332e00b3515dbe0859ceaa30ab66e00Atsushi Nemoto if (--device->privatecnt == 0) 5180f571515c332e00b3515dbe0859ceaa30ab66e00Atsushi Nemoto dma_cap_clear(DMA_PRIVATE, device->cap_mask); 519e2346677af86150c6083974585c131e8a2c3ddccDan Williams chan = NULL; 520e2346677af86150c6083974585c131e8a2c3ddccDan Williams } 52159b5ec21446b9239d706ab237fb261d525b75e81Dan Williams } 52259b5ec21446b9239d706ab237fb261d525b75e81Dan Williams mutex_unlock(&dma_list_mutex); 52359b5ec21446b9239d706ab237fb261d525b75e81Dan Williams 52459b5ec21446b9239d706ab237fb261d525b75e81Dan Williams pr_debug("%s: %s (%s)\n", __func__, chan ? "success" : "fail", 52541d5e59c1299f27983977bcfe3b360600996051cDan Williams chan ? dma_chan_name(chan) : NULL); 52659b5ec21446b9239d706ab237fb261d525b75e81Dan Williams 52759b5ec21446b9239d706ab237fb261d525b75e81Dan Williams return chan; 52859b5ec21446b9239d706ab237fb261d525b75e81Dan Williams} 52959b5ec21446b9239d706ab237fb261d525b75e81Dan WilliamsEXPORT_SYMBOL_GPL(__dma_request_channel); 53059b5ec21446b9239d706ab237fb261d525b75e81Dan Williams 53159b5ec21446b9239d706ab237fb261d525b75e81Dan Williamsvoid dma_release_channel(struct dma_chan *chan) 53259b5ec21446b9239d706ab237fb261d525b75e81Dan Williams{ 53359b5ec21446b9239d706ab237fb261d525b75e81Dan Williams mutex_lock(&dma_list_mutex); 53459b5ec21446b9239d706ab237fb261d525b75e81Dan Williams WARN_ONCE(chan->client_count != 1, 53559b5ec21446b9239d706ab237fb261d525b75e81Dan Williams "chan reference count %d != 1\n", chan->client_count); 53659b5ec21446b9239d706ab237fb261d525b75e81Dan Williams dma_chan_put(chan); 5370f571515c332e00b3515dbe0859ceaa30ab66e00Atsushi Nemoto /* drop PRIVATE cap enabled by __dma_request_channel() */ 5380f571515c332e00b3515dbe0859ceaa30ab66e00Atsushi Nemoto if (--chan->device->privatecnt == 0) 5390f571515c332e00b3515dbe0859ceaa30ab66e00Atsushi Nemoto dma_cap_clear(DMA_PRIVATE, chan->device->cap_mask); 54059b5ec21446b9239d706ab237fb261d525b75e81Dan Williams mutex_unlock(&dma_list_mutex); 54159b5ec21446b9239d706ab237fb261d525b75e81Dan Williams} 54259b5ec21446b9239d706ab237fb261d525b75e81Dan WilliamsEXPORT_SYMBOL_GPL(dma_release_channel); 54359b5ec21446b9239d706ab237fb261d525b75e81Dan Williams 544bec085134e446577a983f17f57d642a88d1af53bDan Williams/** 545209b84a88fe81341b4d8d465acc4a67cb7c3feb3Dan Williams * dmaengine_get - register interest in dma_channels 546d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams */ 547209b84a88fe81341b4d8d465acc4a67cb7c3feb3Dan Williamsvoid dmaengine_get(void) 548d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams{ 5496f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams struct dma_device *device, *_d; 5506f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams struct dma_chan *chan; 5516f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams int err; 5526f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams 553c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech mutex_lock(&dma_list_mutex); 5546f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams dmaengine_ref_count++; 5556f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams 5566f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams /* try to grab channels */ 55759b5ec21446b9239d706ab237fb261d525b75e81Dan Williams list_for_each_entry_safe(device, _d, &dma_device_list, global_node) { 55859b5ec21446b9239d706ab237fb261d525b75e81Dan Williams if (dma_has_cap(DMA_PRIVATE, device->cap_mask)) 55959b5ec21446b9239d706ab237fb261d525b75e81Dan Williams continue; 5606f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams list_for_each_entry(chan, &device->channels, device_node) { 5616f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams err = dma_chan_get(chan); 5626f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams if (err == -ENODEV) { 5636f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams /* module removed before we could use it */ 5642ba05622b8b143b0c95968ba59bddfbd6d2f2559Dan Williams list_del_rcu(&device->global_node); 5656f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams break; 5666f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams } else if (err) 5676f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams pr_err("dmaengine: failed to get %s: (%d)\n", 56841d5e59c1299f27983977bcfe3b360600996051cDan Williams dma_chan_name(chan), err); 5696f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams } 57059b5ec21446b9239d706ab237fb261d525b75e81Dan Williams } 5716f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams 572bec085134e446577a983f17f57d642a88d1af53bDan Williams /* if this is the first reference and there were channels 573bec085134e446577a983f17f57d642a88d1af53bDan Williams * waiting we need to rebalance to get those channels 574bec085134e446577a983f17f57d642a88d1af53bDan Williams * incorporated into the channel table 575bec085134e446577a983f17f57d642a88d1af53bDan Williams */ 576bec085134e446577a983f17f57d642a88d1af53bDan Williams if (dmaengine_ref_count == 1) 577bec085134e446577a983f17f57d642a88d1af53bDan Williams dma_channel_rebalance(); 578c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech mutex_unlock(&dma_list_mutex); 579c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 580209b84a88fe81341b4d8d465acc4a67cb7c3feb3Dan WilliamsEXPORT_SYMBOL(dmaengine_get); 581c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 582c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech/** 583209b84a88fe81341b4d8d465acc4a67cb7c3feb3Dan Williams * dmaengine_put - let dma drivers be removed when ref_count == 0 584c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech */ 585209b84a88fe81341b4d8d465acc4a67cb7c3feb3Dan Williamsvoid dmaengine_put(void) 586c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 587d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams struct dma_device *device; 588c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech struct dma_chan *chan; 589c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 590c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech mutex_lock(&dma_list_mutex); 5916f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams dmaengine_ref_count--; 5926f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams BUG_ON(dmaengine_ref_count < 0); 5936f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams /* drop channel references */ 59459b5ec21446b9239d706ab237fb261d525b75e81Dan Williams list_for_each_entry(device, &dma_device_list, global_node) { 59559b5ec21446b9239d706ab237fb261d525b75e81Dan Williams if (dma_has_cap(DMA_PRIVATE, device->cap_mask)) 59659b5ec21446b9239d706ab237fb261d525b75e81Dan Williams continue; 5976f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams list_for_each_entry(chan, &device->channels, device_node) 5986f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams dma_chan_put(chan); 59959b5ec21446b9239d706ab237fb261d525b75e81Dan Williams } 600c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech mutex_unlock(&dma_list_mutex); 601c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 602209b84a88fe81341b4d8d465acc4a67cb7c3feb3Dan WilliamsEXPORT_SYMBOL(dmaengine_put); 603c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 604138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williamsstatic bool device_has_all_tx_types(struct dma_device *device) 605138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams{ 606138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams /* A device that satisfies this test has channels that will never cause 607138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams * an async_tx channel switch event as all possible operation types can 608138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams * be handled. 609138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams */ 610138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams #ifdef CONFIG_ASYNC_TX_DMA 611138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams if (!dma_has_cap(DMA_INTERRUPT, device->cap_mask)) 612138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams return false; 613138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams #endif 614138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams 615138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams #if defined(CONFIG_ASYNC_MEMCPY) || defined(CONFIG_ASYNC_MEMCPY_MODULE) 616138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams if (!dma_has_cap(DMA_MEMCPY, device->cap_mask)) 617138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams return false; 618138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams #endif 619138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams 620138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams #if defined(CONFIG_ASYNC_MEMSET) || defined(CONFIG_ASYNC_MEMSET_MODULE) 621138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams if (!dma_has_cap(DMA_MEMSET, device->cap_mask)) 622138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams return false; 623138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams #endif 624138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams 625138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams #if defined(CONFIG_ASYNC_XOR) || defined(CONFIG_ASYNC_XOR_MODULE) 626138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams if (!dma_has_cap(DMA_XOR, device->cap_mask)) 627138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams return false; 6287b3cc2b1fc2066391e498f3387204908c4eced21Dan Williams 6297b3cc2b1fc2066391e498f3387204908c4eced21Dan Williams #ifndef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA 6304499a24dec00e037da7d09caccad45e7594a9c19Dan Williams if (!dma_has_cap(DMA_XOR_VAL, device->cap_mask)) 6314499a24dec00e037da7d09caccad45e7594a9c19Dan Williams return false; 632138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams #endif 6337b3cc2b1fc2066391e498f3387204908c4eced21Dan Williams #endif 634138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams 635138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams #if defined(CONFIG_ASYNC_PQ) || defined(CONFIG_ASYNC_PQ_MODULE) 636138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams if (!dma_has_cap(DMA_PQ, device->cap_mask)) 637138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams return false; 6387b3cc2b1fc2066391e498f3387204908c4eced21Dan Williams 6397b3cc2b1fc2066391e498f3387204908c4eced21Dan Williams #ifndef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA 6404499a24dec00e037da7d09caccad45e7594a9c19Dan Williams if (!dma_has_cap(DMA_PQ_VAL, device->cap_mask)) 6414499a24dec00e037da7d09caccad45e7594a9c19Dan Williams return false; 642138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams #endif 6437b3cc2b1fc2066391e498f3387204908c4eced21Dan Williams #endif 644138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams 645138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams return true; 646138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams} 647138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams 648257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williamsstatic int get_dma_id(struct dma_device *device) 649257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams{ 650257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams int rc; 651257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams 652257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams idr_retry: 653257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams if (!idr_pre_get(&dma_idr, GFP_KERNEL)) 654257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams return -ENOMEM; 655257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams mutex_lock(&dma_list_mutex); 656257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams rc = idr_get_new(&dma_idr, NULL, &device->dev_id); 657257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams mutex_unlock(&dma_list_mutex); 658257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams if (rc == -EAGAIN) 659257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams goto idr_retry; 660257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams else if (rc != 0) 661257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams return rc; 662257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams 663257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams return 0; 664257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams} 665257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams 666c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech/** 6676508871eddbbd3e62799f3b6182a6a4fd3ef31d5Randy Dunlap * dma_async_device_register - registers DMA devices found 668c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * @device: &dma_device 669c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech */ 670c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leechint dma_async_device_register(struct dma_device *device) 671c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 672ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik int chancnt = 0, rc; 673c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech struct dma_chan* chan; 674864498aaa9fef69ee166da023d12413a7776342dDan Williams atomic_t *idr_ref; 675c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 676c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech if (!device) 677c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech return -ENODEV; 678c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 6797405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams /* validate device routines */ 6807405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams BUG_ON(dma_has_cap(DMA_MEMCPY, device->cap_mask) && 6817405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams !device->device_prep_dma_memcpy); 6827405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams BUG_ON(dma_has_cap(DMA_XOR, device->cap_mask) && 6837405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams !device->device_prep_dma_xor); 684099f53cb50e45ef617a9f1d63ceec799e489418bDan Williams BUG_ON(dma_has_cap(DMA_XOR_VAL, device->cap_mask) && 685099f53cb50e45ef617a9f1d63ceec799e489418bDan Williams !device->device_prep_dma_xor_val); 686b2f46fd8ef3dff2ab30f31126833f78b7480283aDan Williams BUG_ON(dma_has_cap(DMA_PQ, device->cap_mask) && 687b2f46fd8ef3dff2ab30f31126833f78b7480283aDan Williams !device->device_prep_dma_pq); 688b2f46fd8ef3dff2ab30f31126833f78b7480283aDan Williams BUG_ON(dma_has_cap(DMA_PQ_VAL, device->cap_mask) && 689b2f46fd8ef3dff2ab30f31126833f78b7480283aDan Williams !device->device_prep_dma_pq_val); 6907405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams BUG_ON(dma_has_cap(DMA_MEMSET, device->cap_mask) && 6917405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams !device->device_prep_dma_memset); 6929b941c6660bae673e27c207f1d20d98ef8ecd450Zhang Wei BUG_ON(dma_has_cap(DMA_INTERRUPT, device->cap_mask) && 6937405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams !device->device_prep_dma_interrupt); 694a86ee03ce6f279ebe581a7a8c0c4393eaeb789eeIra Snyder BUG_ON(dma_has_cap(DMA_SG, device->cap_mask) && 695a86ee03ce6f279ebe581a7a8c0c4393eaeb789eeIra Snyder !device->device_prep_dma_sg); 696782bc950d84e404422ba21008fd51ee894c8d231Sascha Hauer BUG_ON(dma_has_cap(DMA_CYCLIC, device->cap_mask) && 697782bc950d84e404422ba21008fd51ee894c8d231Sascha Hauer !device->device_prep_dma_cyclic); 698dc0ee6435cb92ccc81b14ff28d163fecc5a7f120Haavard Skinnemoen BUG_ON(dma_has_cap(DMA_SLAVE, device->cap_mask) && 699c3635c78e500a52c9fcd55de381a72928d9e054dLinus Walleij !device->device_control); 700b14dab792dee3245b628e046d80a7fad5573fea6Jassi Brar BUG_ON(dma_has_cap(DMA_INTERLEAVE, device->cap_mask) && 701b14dab792dee3245b628e046d80a7fad5573fea6Jassi Brar !device->device_prep_interleaved_dma); 7027405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 7037405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams BUG_ON(!device->device_alloc_chan_resources); 7047405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams BUG_ON(!device->device_free_chan_resources); 7050793448187643b50af89d36b08470baf45a3cab4Linus Walleij BUG_ON(!device->device_tx_status); 7067405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams BUG_ON(!device->device_issue_pending); 7077405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams BUG_ON(!device->dev); 7087405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 709138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams /* note: this only matters in the 7105fc6d897fde352bad5db5767e7260741a8cdd9e9Dan Williams * CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH=n case 711138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams */ 712138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams if (device_has_all_tx_types(device)) 713138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams dma_cap_set(DMA_ASYNC_TX, device->cap_mask); 714138f4c359d23d2ec38d18bd70dd9613ae515fe93Dan Williams 715864498aaa9fef69ee166da023d12413a7776342dDan Williams idr_ref = kmalloc(sizeof(*idr_ref), GFP_KERNEL); 716864498aaa9fef69ee166da023d12413a7776342dDan Williams if (!idr_ref) 717864498aaa9fef69ee166da023d12413a7776342dDan Williams return -ENOMEM; 718257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams rc = get_dma_id(device); 719257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams if (rc != 0) { 720257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams kfree(idr_ref); 721864498aaa9fef69ee166da023d12413a7776342dDan Williams return rc; 722257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams } 723257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams 724257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams atomic_set(idr_ref, 0); 725c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 726c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech /* represent channels in sysfs. Probably want devs too */ 727c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech list_for_each_entry(chan, &device->channels, device_node) { 728257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams rc = -ENOMEM; 729c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech chan->local = alloc_percpu(typeof(*chan->local)); 730c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech if (chan->local == NULL) 731257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams goto err_out; 73241d5e59c1299f27983977bcfe3b360600996051cDan Williams chan->dev = kzalloc(sizeof(*chan->dev), GFP_KERNEL); 73341d5e59c1299f27983977bcfe3b360600996051cDan Williams if (chan->dev == NULL) { 73441d5e59c1299f27983977bcfe3b360600996051cDan Williams free_percpu(chan->local); 735257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams chan->local = NULL; 736257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams goto err_out; 73741d5e59c1299f27983977bcfe3b360600996051cDan Williams } 738c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 739c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech chan->chan_id = chancnt++; 74041d5e59c1299f27983977bcfe3b360600996051cDan Williams chan->dev->device.class = &dma_devclass; 74141d5e59c1299f27983977bcfe3b360600996051cDan Williams chan->dev->device.parent = device->dev; 74241d5e59c1299f27983977bcfe3b360600996051cDan Williams chan->dev->chan = chan; 743864498aaa9fef69ee166da023d12413a7776342dDan Williams chan->dev->idr_ref = idr_ref; 744864498aaa9fef69ee166da023d12413a7776342dDan Williams chan->dev->dev_id = device->dev_id; 745864498aaa9fef69ee166da023d12413a7776342dDan Williams atomic_inc(idr_ref); 74641d5e59c1299f27983977bcfe3b360600996051cDan Williams dev_set_name(&chan->dev->device, "dma%dchan%d", 74706190d8415219d9eef7d8f04b52a109e34575a76Kay Sievers device->dev_id, chan->chan_id); 748c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 74941d5e59c1299f27983977bcfe3b360600996051cDan Williams rc = device_register(&chan->dev->device); 750ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik if (rc) { 751ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik free_percpu(chan->local); 752ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik chan->local = NULL; 753257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams kfree(chan->dev); 754257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams atomic_dec(idr_ref); 755ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik goto err_out; 756ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik } 7577cc5bf9a3a84e5a02e23e5739fb894790b37c101Dan Williams chan->client_count = 0; 758c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech } 75959b5ec21446b9239d706ab237fb261d525b75e81Dan Williams device->chancnt = chancnt; 760c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 761c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech mutex_lock(&dma_list_mutex); 76259b5ec21446b9239d706ab237fb261d525b75e81Dan Williams /* take references on public channels */ 76359b5ec21446b9239d706ab237fb261d525b75e81Dan Williams if (dmaengine_ref_count && !dma_has_cap(DMA_PRIVATE, device->cap_mask)) 7646f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams list_for_each_entry(chan, &device->channels, device_node) { 7656f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams /* if clients are already waiting for channels we need 7666f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams * to take references on their behalf 7676f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams */ 7686f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams if (dma_chan_get(chan) == -ENODEV) { 7696f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams /* note we can only get here for the first 7706f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams * channel as the remaining channels are 7716f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams * guaranteed to get a reference 7726f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams */ 7736f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams rc = -ENODEV; 7746f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams mutex_unlock(&dma_list_mutex); 7756f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams goto err_out; 7766f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams } 7776f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams } 7782ba05622b8b143b0c95968ba59bddfbd6d2f2559Dan Williams list_add_tail_rcu(&device->global_node, &dma_device_list); 7790f571515c332e00b3515dbe0859ceaa30ab66e00Atsushi Nemoto if (dma_has_cap(DMA_PRIVATE, device->cap_mask)) 7800f571515c332e00b3515dbe0859ceaa30ab66e00Atsushi Nemoto device->privatecnt++; /* Always private */ 781bec085134e446577a983f17f57d642a88d1af53bDan Williams dma_channel_rebalance(); 782c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech mutex_unlock(&dma_list_mutex); 783c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 784c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech return 0; 785ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik 786ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzikerr_out: 787257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams /* if we never registered a channel just release the idr */ 788257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams if (atomic_read(idr_ref) == 0) { 789257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams mutex_lock(&dma_list_mutex); 790257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams idr_remove(&dma_idr, device->dev_id); 791257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams mutex_unlock(&dma_list_mutex); 792257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams kfree(idr_ref); 793257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams return rc; 794257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams } 795257b17ca030387cb17314cd1851507bdd1b4ddd5Dan Williams 796ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik list_for_each_entry(chan, &device->channels, device_node) { 797ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik if (chan->local == NULL) 798ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik continue; 79941d5e59c1299f27983977bcfe3b360600996051cDan Williams mutex_lock(&dma_list_mutex); 80041d5e59c1299f27983977bcfe3b360600996051cDan Williams chan->dev->chan = NULL; 80141d5e59c1299f27983977bcfe3b360600996051cDan Williams mutex_unlock(&dma_list_mutex); 80241d5e59c1299f27983977bcfe3b360600996051cDan Williams device_unregister(&chan->dev->device); 803ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik free_percpu(chan->local); 804ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik } 805ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik return rc; 806c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 807765e3d8a71bbc1f3400667d5cfcfd7b03382d587David BrownellEXPORT_SYMBOL(dma_async_device_register); 808c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 809c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech/** 8106f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams * dma_async_device_unregister - unregister a DMA device 8116508871eddbbd3e62799f3b6182a6a4fd3ef31d5Randy Dunlap * @device: &dma_device 812f27c580c3628d79b17f38976d842a6d7f3616e2eDan Williams * 813f27c580c3628d79b17f38976d842a6d7f3616e2eDan Williams * This routine is called by dma driver exit routines, dmaengine holds module 814f27c580c3628d79b17f38976d842a6d7f3616e2eDan Williams * references to prevent it being called while channels are in use. 8156508871eddbbd3e62799f3b6182a6a4fd3ef31d5Randy Dunlap */ 8166508871eddbbd3e62799f3b6182a6a4fd3ef31d5Randy Dunlapvoid dma_async_device_unregister(struct dma_device *device) 817c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 818c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech struct dma_chan *chan; 819c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 820c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech mutex_lock(&dma_list_mutex); 8212ba05622b8b143b0c95968ba59bddfbd6d2f2559Dan Williams list_del_rcu(&device->global_node); 822bec085134e446577a983f17f57d642a88d1af53bDan Williams dma_channel_rebalance(); 823c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech mutex_unlock(&dma_list_mutex); 824c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 825c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech list_for_each_entry(chan, &device->channels, device_node) { 8266f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams WARN_ONCE(chan->client_count, 8276f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams "%s called while %d clients hold a reference\n", 8286f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1Dan Williams __func__, chan->client_count); 82941d5e59c1299f27983977bcfe3b360600996051cDan Williams mutex_lock(&dma_list_mutex); 83041d5e59c1299f27983977bcfe3b360600996051cDan Williams chan->dev->chan = NULL; 83141d5e59c1299f27983977bcfe3b360600996051cDan Williams mutex_unlock(&dma_list_mutex); 83241d5e59c1299f27983977bcfe3b360600996051cDan Williams device_unregister(&chan->dev->device); 833adef477268ff5ddd0195611dc7e26d7a879fefe1Anatolij Gustschin free_percpu(chan->local); 834c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech } 835c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 836765e3d8a71bbc1f3400667d5cfcfd7b03382d587David BrownellEXPORT_SYMBOL(dma_async_device_unregister); 837c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 8387405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams/** 8397405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses 8407405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @chan: DMA channel to offload copy to 8417405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @dest: destination address (virtual) 8427405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @src: source address (virtual) 8437405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @len: length 8447405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * 8457405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * Both @dest and @src must be mappable to a bus address according to the 8467405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * DMA mapping API rules for streaming mappings. 8477405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * Both @dest and @src must stay memory resident (kernel memory or locked 8487405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * user space pages). 8497405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams */ 8507405f74badf46b5d023c5d2b670b4471525f6c91Dan Williamsdma_cookie_t 8517405f74badf46b5d023c5d2b670b4471525f6c91Dan Williamsdma_async_memcpy_buf_to_buf(struct dma_chan *chan, void *dest, 8527405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams void *src, size_t len) 8537405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams{ 8547405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams struct dma_device *dev = chan->device; 8557405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams struct dma_async_tx_descriptor *tx; 8560036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_addr_t dma_dest, dma_src; 8577405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams dma_cookie_t cookie; 8584f005dbe5584fe54c9f6d6d4f0acd3fb29be84daMaciej Sosnowski unsigned long flags; 8597405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 8600036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_src = dma_map_single(dev->dev, src, len, DMA_TO_DEVICE); 8610036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_dest = dma_map_single(dev->dev, dest, len, DMA_FROM_DEVICE); 8624f005dbe5584fe54c9f6d6d4f0acd3fb29be84daMaciej Sosnowski flags = DMA_CTRL_ACK | 8634f005dbe5584fe54c9f6d6d4f0acd3fb29be84daMaciej Sosnowski DMA_COMPL_SRC_UNMAP_SINGLE | 8644f005dbe5584fe54c9f6d6d4f0acd3fb29be84daMaciej Sosnowski DMA_COMPL_DEST_UNMAP_SINGLE; 8654f005dbe5584fe54c9f6d6d4f0acd3fb29be84daMaciej Sosnowski tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, flags); 8660036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams 8670036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams if (!tx) { 8680036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE); 8690036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_unmap_single(dev->dev, dma_dest, len, DMA_FROM_DEVICE); 8707405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams return -ENOMEM; 8710036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams } 8727405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 8737405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams tx->callback = NULL; 8747405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams cookie = tx->tx_submit(tx); 8757405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 876e7dcaa4755e35d7540bf19f316f8798357c53fa0Christoph Lameter preempt_disable(); 877e7dcaa4755e35d7540bf19f316f8798357c53fa0Christoph Lameter __this_cpu_add(chan->local->bytes_transferred, len); 878e7dcaa4755e35d7540bf19f316f8798357c53fa0Christoph Lameter __this_cpu_inc(chan->local->memcpy_count); 879e7dcaa4755e35d7540bf19f316f8798357c53fa0Christoph Lameter preempt_enable(); 8807405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 8817405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams return cookie; 8827405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams} 8837405f74badf46b5d023c5d2b670b4471525f6c91Dan WilliamsEXPORT_SYMBOL(dma_async_memcpy_buf_to_buf); 8847405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 8857405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams/** 8867405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * dma_async_memcpy_buf_to_pg - offloaded copy from address to page 8877405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @chan: DMA channel to offload copy to 8887405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @page: destination page 8897405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @offset: offset in page to copy to 8907405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @kdata: source address (virtual) 8917405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @len: length 8927405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * 8937405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * Both @page/@offset and @kdata must be mappable to a bus address according 8947405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * to the DMA mapping API rules for streaming mappings. 8957405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * Both @page/@offset and @kdata must stay memory resident (kernel memory or 8967405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * locked user space pages) 8977405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams */ 8987405f74badf46b5d023c5d2b670b4471525f6c91Dan Williamsdma_cookie_t 8997405f74badf46b5d023c5d2b670b4471525f6c91Dan Williamsdma_async_memcpy_buf_to_pg(struct dma_chan *chan, struct page *page, 9007405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams unsigned int offset, void *kdata, size_t len) 9017405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams{ 9027405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams struct dma_device *dev = chan->device; 9037405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams struct dma_async_tx_descriptor *tx; 9040036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_addr_t dma_dest, dma_src; 9057405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams dma_cookie_t cookie; 9064f005dbe5584fe54c9f6d6d4f0acd3fb29be84daMaciej Sosnowski unsigned long flags; 9077405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 9080036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_src = dma_map_single(dev->dev, kdata, len, DMA_TO_DEVICE); 9090036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_dest = dma_map_page(dev->dev, page, offset, len, DMA_FROM_DEVICE); 9104f005dbe5584fe54c9f6d6d4f0acd3fb29be84daMaciej Sosnowski flags = DMA_CTRL_ACK | DMA_COMPL_SRC_UNMAP_SINGLE; 9114f005dbe5584fe54c9f6d6d4f0acd3fb29be84daMaciej Sosnowski tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, flags); 9120036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams 9130036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams if (!tx) { 9140036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE); 9150036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_unmap_page(dev->dev, dma_dest, len, DMA_FROM_DEVICE); 9167405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams return -ENOMEM; 9170036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams } 9187405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 9197405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams tx->callback = NULL; 9207405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams cookie = tx->tx_submit(tx); 9217405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 922e7dcaa4755e35d7540bf19f316f8798357c53fa0Christoph Lameter preempt_disable(); 923e7dcaa4755e35d7540bf19f316f8798357c53fa0Christoph Lameter __this_cpu_add(chan->local->bytes_transferred, len); 924e7dcaa4755e35d7540bf19f316f8798357c53fa0Christoph Lameter __this_cpu_inc(chan->local->memcpy_count); 925e7dcaa4755e35d7540bf19f316f8798357c53fa0Christoph Lameter preempt_enable(); 9267405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 9277405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams return cookie; 9287405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams} 9297405f74badf46b5d023c5d2b670b4471525f6c91Dan WilliamsEXPORT_SYMBOL(dma_async_memcpy_buf_to_pg); 9307405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 9317405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams/** 9327405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * dma_async_memcpy_pg_to_pg - offloaded copy from page to page 9337405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @chan: DMA channel to offload copy to 9347405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @dest_pg: destination page 9357405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @dest_off: offset in page to copy to 9367405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @src_pg: source page 9377405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @src_off: offset in page to copy from 9387405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @len: length 9397405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * 9407405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * Both @dest_page/@dest_off and @src_page/@src_off must be mappable to a bus 9417405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * address according to the DMA mapping API rules for streaming mappings. 9427405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * Both @dest_page/@dest_off and @src_page/@src_off must stay memory resident 9437405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * (kernel memory or locked user space pages). 9447405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams */ 9457405f74badf46b5d023c5d2b670b4471525f6c91Dan Williamsdma_cookie_t 9467405f74badf46b5d023c5d2b670b4471525f6c91Dan Williamsdma_async_memcpy_pg_to_pg(struct dma_chan *chan, struct page *dest_pg, 9477405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams unsigned int dest_off, struct page *src_pg, unsigned int src_off, 9487405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams size_t len) 9497405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams{ 9507405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams struct dma_device *dev = chan->device; 9517405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams struct dma_async_tx_descriptor *tx; 9520036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_addr_t dma_dest, dma_src; 9537405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams dma_cookie_t cookie; 9544f005dbe5584fe54c9f6d6d4f0acd3fb29be84daMaciej Sosnowski unsigned long flags; 9557405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 9560036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_src = dma_map_page(dev->dev, src_pg, src_off, len, DMA_TO_DEVICE); 9570036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_dest = dma_map_page(dev->dev, dest_pg, dest_off, len, 9580036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams DMA_FROM_DEVICE); 9594f005dbe5584fe54c9f6d6d4f0acd3fb29be84daMaciej Sosnowski flags = DMA_CTRL_ACK; 9604f005dbe5584fe54c9f6d6d4f0acd3fb29be84daMaciej Sosnowski tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, flags); 9610036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams 9620036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams if (!tx) { 9630036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_unmap_page(dev->dev, dma_src, len, DMA_TO_DEVICE); 9640036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_unmap_page(dev->dev, dma_dest, len, DMA_FROM_DEVICE); 9657405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams return -ENOMEM; 9660036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams } 9677405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 9687405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams tx->callback = NULL; 9697405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams cookie = tx->tx_submit(tx); 9707405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 971e7dcaa4755e35d7540bf19f316f8798357c53fa0Christoph Lameter preempt_disable(); 972e7dcaa4755e35d7540bf19f316f8798357c53fa0Christoph Lameter __this_cpu_add(chan->local->bytes_transferred, len); 973e7dcaa4755e35d7540bf19f316f8798357c53fa0Christoph Lameter __this_cpu_inc(chan->local->memcpy_count); 974e7dcaa4755e35d7540bf19f316f8798357c53fa0Christoph Lameter preempt_enable(); 9757405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 9767405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams return cookie; 9777405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams} 9787405f74badf46b5d023c5d2b670b4471525f6c91Dan WilliamsEXPORT_SYMBOL(dma_async_memcpy_pg_to_pg); 9797405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 9807405f74badf46b5d023c5d2b670b4471525f6c91Dan Williamsvoid dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx, 9817405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams struct dma_chan *chan) 9827405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams{ 9837405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams tx->chan = chan; 9845fc6d897fde352bad5db5767e7260741a8cdd9e9Dan Williams #ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH 9857405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams spin_lock_init(&tx->lock); 986caa20d974c86af496b419eef70010e63b7fab7acDan Williams #endif 9877405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams} 9887405f74badf46b5d023c5d2b670b4471525f6c91Dan WilliamsEXPORT_SYMBOL(dma_async_tx_descriptor_init); 9897405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 99007f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams/* dma_wait_for_async_tx - spin wait for a transaction to complete 99107f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams * @tx: in-flight transaction to wait on 99207f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams */ 99307f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williamsenum dma_status 99407f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williamsdma_wait_for_async_tx(struct dma_async_tx_descriptor *tx) 99507f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams{ 99695475e57113c66aac7583925736ed2e2d58c990dDan Williams unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000); 99707f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams 99807f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams if (!tx) 99907f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams return DMA_SUCCESS; 100007f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams 100195475e57113c66aac7583925736ed2e2d58c990dDan Williams while (tx->cookie == -EBUSY) { 100295475e57113c66aac7583925736ed2e2d58c990dDan Williams if (time_after_eq(jiffies, dma_sync_wait_timeout)) { 100395475e57113c66aac7583925736ed2e2d58c990dDan Williams pr_err("%s timeout waiting for descriptor submission\n", 100495475e57113c66aac7583925736ed2e2d58c990dDan Williams __func__); 100595475e57113c66aac7583925736ed2e2d58c990dDan Williams return DMA_ERROR; 100695475e57113c66aac7583925736ed2e2d58c990dDan Williams } 100795475e57113c66aac7583925736ed2e2d58c990dDan Williams cpu_relax(); 100895475e57113c66aac7583925736ed2e2d58c990dDan Williams } 100995475e57113c66aac7583925736ed2e2d58c990dDan Williams return dma_sync_wait(tx->chan, tx->cookie); 101007f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams} 101107f2211e4fbce6990722d78c4f04225da9c0e9cfDan WilliamsEXPORT_SYMBOL_GPL(dma_wait_for_async_tx); 101207f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams 101307f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams/* dma_run_dependencies - helper routine for dma drivers to process 101407f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams * (start) dependent operations on their target channel 101507f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams * @tx: transaction with dependencies 101607f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams */ 101707f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williamsvoid dma_run_dependencies(struct dma_async_tx_descriptor *tx) 101807f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams{ 1019caa20d974c86af496b419eef70010e63b7fab7acDan Williams struct dma_async_tx_descriptor *dep = txd_next(tx); 102007f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams struct dma_async_tx_descriptor *dep_next; 102107f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams struct dma_chan *chan; 102207f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams 102307f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams if (!dep) 102407f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams return; 102507f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams 1026dd59b8537f6cb53ab863fafad86a5828f1e889a2Yuri Tikhonov /* we'll submit tx->next now, so clear the link */ 1027caa20d974c86af496b419eef70010e63b7fab7acDan Williams txd_clear_next(tx); 102807f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams chan = dep->chan; 102907f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams 103007f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams /* keep submitting up until a channel switch is detected 103107f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams * in that case we will be called again as a result of 103207f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams * processing the interrupt from async_tx_channel_switch 103307f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams */ 103407f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams for (; dep; dep = dep_next) { 1035caa20d974c86af496b419eef70010e63b7fab7acDan Williams txd_lock(dep); 1036caa20d974c86af496b419eef70010e63b7fab7acDan Williams txd_clear_parent(dep); 1037caa20d974c86af496b419eef70010e63b7fab7acDan Williams dep_next = txd_next(dep); 103807f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams if (dep_next && dep_next->chan == chan) 1039caa20d974c86af496b419eef70010e63b7fab7acDan Williams txd_clear_next(dep); /* ->next will be submitted */ 104007f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams else 104107f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams dep_next = NULL; /* submit current dep and terminate */ 1042caa20d974c86af496b419eef70010e63b7fab7acDan Williams txd_unlock(dep); 104307f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams 104407f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams dep->tx_submit(dep); 104507f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams } 104607f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams 104707f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams chan->device->device_issue_pending(chan); 104807f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams} 104907f2211e4fbce6990722d78c4f04225da9c0e9cfDan WilliamsEXPORT_SYMBOL_GPL(dma_run_dependencies); 105007f2211e4fbce6990722d78c4f04225da9c0e9cfDan Williams 1051c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leechstatic int __init dma_bus_init(void) 1052c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 1053c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech return class_register(&dma_devclass); 1054c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 1055652afc27b26859a0ea5f6db681d80b83d2c43cf8Dan Williamsarch_initcall(dma_bus_init); 1056c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 1057bec085134e446577a983f17f57d642a88d1af53bDan Williams 1058