dmaengine.c revision 0036731c88fdb5bf4f04a796a30b5e445fc57f54
1c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech/* 2c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. 3c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * 4c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * This program is free software; you can redistribute it and/or modify it 5c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * under the terms of the GNU General Public License as published by the Free 6c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * Software Foundation; either version 2 of the License, or (at your option) 7c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * any later version. 8c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * 9c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * This program is distributed in the hope that it will be useful, but WITHOUT 10c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * more details. 13c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * 14c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * You should have received a copy of the GNU General Public License along with 15c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * this program; if not, write to the Free Software Foundation, Inc., 59 16c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * 18c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * The full GNU General Public License is included in this distribution in the 19c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * file called COPYING. 20c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech */ 21c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 22c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech/* 23c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * This code implements the DMA subsystem. It provides a HW-neutral interface 24c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * for other kernel code to use asynchronous memory copy capabilities, 25c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * if present, and allows different HW DMA drivers to register as providing 26c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * this capability. 27c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * 28c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * Due to the fact we are accelerating what is already a relatively fast 29c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * operation, the code goes to great lengths to avoid additional overhead, 30c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * such as locking. 31c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * 32c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * LOCKING: 33c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * 34c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * The subsystem keeps two global lists, dma_device_list and dma_client_list. 35c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * Both of these are protected by a mutex, dma_list_mutex. 36c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * 37c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * Each device has a channels list, which runs unlocked but is never modified 38c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * once the device is registered, it's just setup by the driver. 39c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * 40d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams * Each client is responsible for keeping track of the channels it uses. See 41d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams * the definition of dma_event_callback in dmaengine.h. 42c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * 43c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * Each device has a kref, which is initialized to 1 when the device is 44891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jones * registered. A kref_get is done for each device registered. When the 45891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jones * device is released, the coresponding kref_put is done in the release 46c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * method. Every time one of the device's channels is allocated to a client, 47c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * a kref_get occurs. When the channel is freed, the coresponding kref_put 48c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * happens. The device's release function does a completion, so 49891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jones * unregister_device does a remove event, device_unregister, a kref_put 50c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * for the first reference, then waits on the completion for all other 51c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * references to finish. 52c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * 53c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * Each channel has an open-coded implementation of Rusty Russell's "bigref," 54d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams * with a kref and a per_cpu local_t. A dma_chan_get is called when a client 55d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams * signals that it wants to use a channel, and dma_chan_put is called when 56d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams * a channel is removed or a client using it is unregesitered. A client can 57d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams * take extra references per outstanding transaction, as is the case with 58d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams * the NET DMA client. The release function does a kref_put on the device. 59d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams * -ChrisL, DanW 60c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech */ 61c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 62c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech#include <linux/init.h> 63c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech#include <linux/module.h> 647405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams#include <linux/mm.h> 65c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech#include <linux/device.h> 66c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech#include <linux/dmaengine.h> 67c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech#include <linux/hardirq.h> 68c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech#include <linux/spinlock.h> 69c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech#include <linux/percpu.h> 70c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech#include <linux/rcupdate.h> 71c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech#include <linux/mutex.h> 727405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams#include <linux/jiffies.h> 73c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 74c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leechstatic DEFINE_MUTEX(dma_list_mutex); 75c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leechstatic LIST_HEAD(dma_device_list); 76c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leechstatic LIST_HEAD(dma_client_list); 77c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 78c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech/* --- sysfs implementation --- */ 79c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 80891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jonesstatic ssize_t show_memcpy_count(struct device *dev, struct device_attribute *attr, char *buf) 81c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 82891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jones struct dma_chan *chan = to_dma_chan(dev); 83c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech unsigned long count = 0; 84c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech int i; 85c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 8617f3ae08b6e7fd778371f2cafbd1c988a67ee343Andrew Morton for_each_possible_cpu(i) 87c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech count += per_cpu_ptr(chan->local, i)->memcpy_count; 88c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 89c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech return sprintf(buf, "%lu\n", count); 90c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 91c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 92891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jonesstatic ssize_t show_bytes_transferred(struct device *dev, struct device_attribute *attr, 93891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jones char *buf) 94c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 95891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jones struct dma_chan *chan = to_dma_chan(dev); 96c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech unsigned long count = 0; 97c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech int i; 98c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 9917f3ae08b6e7fd778371f2cafbd1c988a67ee343Andrew Morton for_each_possible_cpu(i) 100c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech count += per_cpu_ptr(chan->local, i)->bytes_transferred; 101c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 102c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech return sprintf(buf, "%lu\n", count); 103c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 104c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 105891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jonesstatic ssize_t show_in_use(struct device *dev, struct device_attribute *attr, char *buf) 106c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 107891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jones struct dma_chan *chan = to_dma_chan(dev); 108d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams int in_use = 0; 109d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams 110d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams if (unlikely(chan->slow_ref) && 111d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams atomic_read(&chan->refcount.refcount) > 1) 112d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams in_use = 1; 113d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams else { 114d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams if (local_read(&(per_cpu_ptr(chan->local, 115d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams get_cpu())->refcount)) > 0) 116d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams in_use = 1; 117d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams put_cpu(); 118d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams } 119c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 120d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams return sprintf(buf, "%d\n", in_use); 121c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 122c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 123891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jonesstatic struct device_attribute dma_attrs[] = { 124c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech __ATTR(memcpy_count, S_IRUGO, show_memcpy_count, NULL), 125c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech __ATTR(bytes_transferred, S_IRUGO, show_bytes_transferred, NULL), 126c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech __ATTR(in_use, S_IRUGO, show_in_use, NULL), 127c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech __ATTR_NULL 128c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech}; 129c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 130c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leechstatic void dma_async_device_cleanup(struct kref *kref); 131c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 132891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jonesstatic void dma_dev_release(struct device *dev) 133c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 134891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jones struct dma_chan *chan = to_dma_chan(dev); 135c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech kref_put(&chan->device->refcount, dma_async_device_cleanup); 136c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 137c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 138c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leechstatic struct class dma_devclass = { 139891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jones .name = "dma", 140891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jones .dev_attrs = dma_attrs, 141891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jones .dev_release = dma_dev_release, 142c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech}; 143c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 144c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech/* --- client and device registration --- */ 145c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 146d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams#define dma_chan_satisfies_mask(chan, mask) \ 147d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams __dma_chan_satisfies_mask((chan), &(mask)) 148d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williamsstatic int 149d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams__dma_chan_satisfies_mask(struct dma_chan *chan, dma_cap_mask_t *want) 150d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams{ 151d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams dma_cap_mask_t has; 152d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams 153d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams bitmap_and(has.bits, want->bits, chan->device->cap_mask.bits, 154d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams DMA_TX_TYPE_END); 155d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END); 156d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams} 157d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams 158c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech/** 159d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams * dma_client_chan_alloc - try to allocate channels to a client 160c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * @client: &dma_client 161c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * 162c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * Called with dma_list_mutex held. 163c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech */ 164d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williamsstatic void dma_client_chan_alloc(struct dma_client *client) 165c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 166c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech struct dma_device *device; 167c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech struct dma_chan *chan; 168c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech int desc; /* allocated descriptor count */ 169d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams enum dma_state_client ack; 170c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 171d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams /* Find a channel */ 172d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams list_for_each_entry(device, &dma_device_list, global_node) 173c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech list_for_each_entry(chan, &device->channels, device_node) { 174d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams if (!dma_chan_satisfies_mask(chan, client->cap_mask)) 175c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech continue; 176c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 177c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech desc = chan->device->device_alloc_chan_resources(chan); 178c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech if (desc >= 0) { 179d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams ack = client->event_callback(client, 180d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams chan, 181d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams DMA_RESOURCE_AVAILABLE); 182d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams 183d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams /* we are done once this client rejects 184d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams * an available resource 185d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams */ 186348badf1e825323c419dd118f65783db0f7d2ec8Haavard Skinnemoen if (ack == DMA_ACK) 187d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams dma_chan_get(chan); 188348badf1e825323c419dd118f65783db0f7d2ec8Haavard Skinnemoen else if (ack == DMA_NAK) 189d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams return; 190c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech } 191c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech } 192c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 193c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 1947405f74badf46b5d023c5d2b670b4471525f6c91Dan Williamsenum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie) 1957405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams{ 1967405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams enum dma_status status; 1977405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000); 1987405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 1997405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams dma_async_issue_pending(chan); 2007405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams do { 2017405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams status = dma_async_is_tx_complete(chan, cookie, NULL, NULL); 2027405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams if (time_after_eq(jiffies, dma_sync_wait_timeout)) { 2037405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams printk(KERN_ERR "dma_sync_wait_timeout!\n"); 2047405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams return DMA_ERROR; 2057405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams } 2067405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams } while (status == DMA_IN_PROGRESS); 2077405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 2087405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams return status; 2097405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams} 2107405f74badf46b5d023c5d2b670b4471525f6c91Dan WilliamsEXPORT_SYMBOL(dma_sync_wait); 2117405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 212c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech/** 2136508871eddbbd3e62799f3b6182a6a4fd3ef31d5Randy Dunlap * dma_chan_cleanup - release a DMA channel's resources 2146508871eddbbd3e62799f3b6182a6a4fd3ef31d5Randy Dunlap * @kref: kernel reference structure that contains the DMA channel device 215c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech */ 216c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leechvoid dma_chan_cleanup(struct kref *kref) 217c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 218c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech struct dma_chan *chan = container_of(kref, struct dma_chan, refcount); 219c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech chan->device->device_free_chan_resources(chan); 220c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech kref_put(&chan->device->refcount, dma_async_device_cleanup); 221c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 222765e3d8a71bbc1f3400667d5cfcfd7b03382d587David BrownellEXPORT_SYMBOL(dma_chan_cleanup); 223c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 224c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leechstatic void dma_chan_free_rcu(struct rcu_head *rcu) 225c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 226c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech struct dma_chan *chan = container_of(rcu, struct dma_chan, rcu); 227c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech int bias = 0x7FFFFFFF; 228c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech int i; 22917f3ae08b6e7fd778371f2cafbd1c988a67ee343Andrew Morton for_each_possible_cpu(i) 230c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech bias -= local_read(&per_cpu_ptr(chan->local, i)->refcount); 231c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech atomic_sub(bias, &chan->refcount.refcount); 232c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech kref_put(&chan->refcount, dma_chan_cleanup); 233c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 234c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 235d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williamsstatic void dma_chan_release(struct dma_chan *chan) 236c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 237c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech atomic_add(0x7FFFFFFF, &chan->refcount.refcount); 238c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech chan->slow_ref = 1; 239c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech call_rcu(&chan->rcu, dma_chan_free_rcu); 240c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 241c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 242c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech/** 243d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams * dma_chans_notify_available - broadcast available channels to the clients 244c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech */ 245d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williamsstatic void dma_clients_notify_available(void) 246c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 247c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech struct dma_client *client; 248c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 249c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech mutex_lock(&dma_list_mutex); 250c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 251d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams list_for_each_entry(client, &dma_client_list, global_node) 252d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams dma_client_chan_alloc(client); 253c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 254c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech mutex_unlock(&dma_list_mutex); 255c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 256c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 257c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech/** 258d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams * dma_chans_notify_available - tell the clients that a channel is going away 259d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams * @chan: channel on its way out 260c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech */ 261d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williamsstatic void dma_clients_notify_removed(struct dma_chan *chan) 262c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 263c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech struct dma_client *client; 264d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams enum dma_state_client ack; 265c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 266d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams mutex_lock(&dma_list_mutex); 267d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams 268d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams list_for_each_entry(client, &dma_client_list, global_node) { 269d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams ack = client->event_callback(client, chan, 270d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams DMA_RESOURCE_REMOVED); 271d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams 272d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams /* client was holding resources for this channel so 273d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams * free it 274d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams */ 275348badf1e825323c419dd118f65783db0f7d2ec8Haavard Skinnemoen if (ack == DMA_ACK) 276d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams dma_chan_put(chan); 277d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams } 278c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 279d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams mutex_unlock(&dma_list_mutex); 280d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams} 281c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 282d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams/** 283d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams * dma_async_client_register - register a &dma_client 284d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams * @client: ptr to a client structure with valid 'event_callback' and 'cap_mask' 285d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams */ 286d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williamsvoid dma_async_client_register(struct dma_client *client) 287d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams{ 288c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech mutex_lock(&dma_list_mutex); 289c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech list_add_tail(&client->global_node, &dma_client_list); 290c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech mutex_unlock(&dma_list_mutex); 291c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 292765e3d8a71bbc1f3400667d5cfcfd7b03382d587David BrownellEXPORT_SYMBOL(dma_async_client_register); 293c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 294c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech/** 295c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * dma_async_client_unregister - unregister a client and free the &dma_client 2966508871eddbbd3e62799f3b6182a6a4fd3ef31d5Randy Dunlap * @client: &dma_client to free 297c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * 298c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * Force frees any allocated DMA channels, frees the &dma_client memory 299c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech */ 300c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leechvoid dma_async_client_unregister(struct dma_client *client) 301c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 302d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams struct dma_device *device; 303c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech struct dma_chan *chan; 304d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams enum dma_state_client ack; 305c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 306c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech if (!client) 307c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech return; 308c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 309c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech mutex_lock(&dma_list_mutex); 310d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams /* free all channels the client is holding */ 311d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams list_for_each_entry(device, &dma_device_list, global_node) 312d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams list_for_each_entry(chan, &device->channels, device_node) { 313d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams ack = client->event_callback(client, chan, 314d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams DMA_RESOURCE_REMOVED); 315d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams 316348badf1e825323c419dd118f65783db0f7d2ec8Haavard Skinnemoen if (ack == DMA_ACK) 317d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams dma_chan_put(chan); 318d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams } 319d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams 320c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech list_del(&client->global_node); 321c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech mutex_unlock(&dma_list_mutex); 322c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 323765e3d8a71bbc1f3400667d5cfcfd7b03382d587David BrownellEXPORT_SYMBOL(dma_async_client_unregister); 324c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 325c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech/** 326d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams * dma_async_client_chan_request - send all available channels to the 327d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams * client that satisfy the capability mask 328d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams * @client - requester 329c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech */ 330d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williamsvoid dma_async_client_chan_request(struct dma_client *client) 331c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 332d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams mutex_lock(&dma_list_mutex); 333d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams dma_client_chan_alloc(client); 334d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams mutex_unlock(&dma_list_mutex); 335c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 336765e3d8a71bbc1f3400667d5cfcfd7b03382d587David BrownellEXPORT_SYMBOL(dma_async_client_chan_request); 337c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 338c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech/** 3396508871eddbbd3e62799f3b6182a6a4fd3ef31d5Randy Dunlap * dma_async_device_register - registers DMA devices found 340c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech * @device: &dma_device 341c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech */ 342c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leechint dma_async_device_register(struct dma_device *device) 343c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 344c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech static int id; 345ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik int chancnt = 0, rc; 346c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech struct dma_chan* chan; 347c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 348c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech if (!device) 349c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech return -ENODEV; 350c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 3517405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams /* validate device routines */ 3527405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams BUG_ON(dma_has_cap(DMA_MEMCPY, device->cap_mask) && 3537405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams !device->device_prep_dma_memcpy); 3547405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams BUG_ON(dma_has_cap(DMA_XOR, device->cap_mask) && 3557405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams !device->device_prep_dma_xor); 3567405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams BUG_ON(dma_has_cap(DMA_ZERO_SUM, device->cap_mask) && 3577405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams !device->device_prep_dma_zero_sum); 3587405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams BUG_ON(dma_has_cap(DMA_MEMSET, device->cap_mask) && 3597405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams !device->device_prep_dma_memset); 3607405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams BUG_ON(dma_has_cap(DMA_ZERO_SUM, device->cap_mask) && 3617405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams !device->device_prep_dma_interrupt); 3627405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 3637405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams BUG_ON(!device->device_alloc_chan_resources); 3647405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams BUG_ON(!device->device_free_chan_resources); 3657405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams BUG_ON(!device->device_dependency_added); 3667405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams BUG_ON(!device->device_is_tx_complete); 3677405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams BUG_ON(!device->device_issue_pending); 3687405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams BUG_ON(!device->dev); 3697405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 370c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech init_completion(&device->done); 371c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech kref_init(&device->refcount); 372c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech device->dev_id = id++; 373c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 374c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech /* represent channels in sysfs. Probably want devs too */ 375c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech list_for_each_entry(chan, &device->channels, device_node) { 376c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech chan->local = alloc_percpu(typeof(*chan->local)); 377c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech if (chan->local == NULL) 378c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech continue; 379c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 380c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech chan->chan_id = chancnt++; 381891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jones chan->dev.class = &dma_devclass; 382891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jones chan->dev.parent = NULL; 383891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jones snprintf(chan->dev.bus_id, BUS_ID_SIZE, "dma%dchan%d", 384c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech device->dev_id, chan->chan_id); 385c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 386891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jones rc = device_register(&chan->dev); 387ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik if (rc) { 388ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik chancnt--; 389ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik free_percpu(chan->local); 390ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik chan->local = NULL; 391ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik goto err_out; 392ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik } 393ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik 394348badf1e825323c419dd118f65783db0f7d2ec8Haavard Skinnemoen /* One for the channel, one of the class device */ 395348badf1e825323c419dd118f65783db0f7d2ec8Haavard Skinnemoen kref_get(&device->refcount); 396c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech kref_get(&device->refcount); 397d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams kref_init(&chan->refcount); 398d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams chan->slow_ref = 0; 399d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams INIT_RCU_HEAD(&chan->rcu); 400c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech } 401c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 402c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech mutex_lock(&dma_list_mutex); 403c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech list_add_tail(&device->global_node, &dma_device_list); 404c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech mutex_unlock(&dma_list_mutex); 405c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 406d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams dma_clients_notify_available(); 407c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 408c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech return 0; 409ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik 410ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzikerr_out: 411ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik list_for_each_entry(chan, &device->channels, device_node) { 412ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik if (chan->local == NULL) 413ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik continue; 414ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik kref_put(&device->refcount, dma_async_device_cleanup); 415891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jones device_unregister(&chan->dev); 416ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik chancnt--; 417ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik free_percpu(chan->local); 418ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik } 419ff487fb773749124550a5ad2b7fbfe0376af6f0dJeff Garzik return rc; 420c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 421765e3d8a71bbc1f3400667d5cfcfd7b03382d587David BrownellEXPORT_SYMBOL(dma_async_device_register); 422c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 423c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech/** 4246508871eddbbd3e62799f3b6182a6a4fd3ef31d5Randy Dunlap * dma_async_device_cleanup - function called when all references are released 4256508871eddbbd3e62799f3b6182a6a4fd3ef31d5Randy Dunlap * @kref: kernel reference object 426c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech */ 427c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leechstatic void dma_async_device_cleanup(struct kref *kref) 428c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 429c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech struct dma_device *device; 430c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 431c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech device = container_of(kref, struct dma_device, refcount); 432c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech complete(&device->done); 433c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 434c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 4356508871eddbbd3e62799f3b6182a6a4fd3ef31d5Randy Dunlap/** 4366508871eddbbd3e62799f3b6182a6a4fd3ef31d5Randy Dunlap * dma_async_device_unregister - unregisters DMA devices 4376508871eddbbd3e62799f3b6182a6a4fd3ef31d5Randy Dunlap * @device: &dma_device 4386508871eddbbd3e62799f3b6182a6a4fd3ef31d5Randy Dunlap */ 4396508871eddbbd3e62799f3b6182a6a4fd3ef31d5Randy Dunlapvoid dma_async_device_unregister(struct dma_device *device) 440c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 441c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech struct dma_chan *chan; 442c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 443c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech mutex_lock(&dma_list_mutex); 444c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech list_del(&device->global_node); 445c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech mutex_unlock(&dma_list_mutex); 446c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 447c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech list_for_each_entry(chan, &device->channels, device_node) { 448d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams dma_clients_notify_removed(chan); 449891f78ea833edd4a1e524e15bfe297a7a84d81a0Tony Jones device_unregister(&chan->dev); 450d379b01e9087a582d58f4b678208a4f8d8376fe7Dan Williams dma_chan_release(chan); 451c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech } 452c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 453c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech kref_put(&device->refcount, dma_async_device_cleanup); 454c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech wait_for_completion(&device->done); 455c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 456765e3d8a71bbc1f3400667d5cfcfd7b03382d587David BrownellEXPORT_SYMBOL(dma_async_device_unregister); 457c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 4587405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams/** 4597405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses 4607405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @chan: DMA channel to offload copy to 4617405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @dest: destination address (virtual) 4627405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @src: source address (virtual) 4637405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @len: length 4647405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * 4657405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * Both @dest and @src must be mappable to a bus address according to the 4667405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * DMA mapping API rules for streaming mappings. 4677405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * Both @dest and @src must stay memory resident (kernel memory or locked 4687405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * user space pages). 4697405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams */ 4707405f74badf46b5d023c5d2b670b4471525f6c91Dan Williamsdma_cookie_t 4717405f74badf46b5d023c5d2b670b4471525f6c91Dan Williamsdma_async_memcpy_buf_to_buf(struct dma_chan *chan, void *dest, 4727405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams void *src, size_t len) 4737405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams{ 4747405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams struct dma_device *dev = chan->device; 4757405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams struct dma_async_tx_descriptor *tx; 4760036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_addr_t dma_dest, dma_src; 4777405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams dma_cookie_t cookie; 4787405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams int cpu; 4797405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 4800036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_src = dma_map_single(dev->dev, src, len, DMA_TO_DEVICE); 4810036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_dest = dma_map_single(dev->dev, dest, len, DMA_FROM_DEVICE); 4820036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, 0); 4830036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams 4840036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams if (!tx) { 4850036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE); 4860036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_unmap_single(dev->dev, dma_dest, len, DMA_FROM_DEVICE); 4877405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams return -ENOMEM; 4880036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams } 4897405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 4907405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams tx->ack = 1; 4917405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams tx->callback = NULL; 4927405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams cookie = tx->tx_submit(tx); 4937405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 4947405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams cpu = get_cpu(); 4957405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; 4967405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams per_cpu_ptr(chan->local, cpu)->memcpy_count++; 4977405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams put_cpu(); 4987405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 4997405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams return cookie; 5007405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams} 5017405f74badf46b5d023c5d2b670b4471525f6c91Dan WilliamsEXPORT_SYMBOL(dma_async_memcpy_buf_to_buf); 5027405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 5037405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams/** 5047405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * dma_async_memcpy_buf_to_pg - offloaded copy from address to page 5057405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @chan: DMA channel to offload copy to 5067405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @page: destination page 5077405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @offset: offset in page to copy to 5087405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @kdata: source address (virtual) 5097405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @len: length 5107405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * 5117405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * Both @page/@offset and @kdata must be mappable to a bus address according 5127405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * to the DMA mapping API rules for streaming mappings. 5137405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * Both @page/@offset and @kdata must stay memory resident (kernel memory or 5147405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * locked user space pages) 5157405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams */ 5167405f74badf46b5d023c5d2b670b4471525f6c91Dan Williamsdma_cookie_t 5177405f74badf46b5d023c5d2b670b4471525f6c91Dan Williamsdma_async_memcpy_buf_to_pg(struct dma_chan *chan, struct page *page, 5187405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams unsigned int offset, void *kdata, size_t len) 5197405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams{ 5207405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams struct dma_device *dev = chan->device; 5217405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams struct dma_async_tx_descriptor *tx; 5220036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_addr_t dma_dest, dma_src; 5237405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams dma_cookie_t cookie; 5247405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams int cpu; 5257405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 5260036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_src = dma_map_single(dev->dev, kdata, len, DMA_TO_DEVICE); 5270036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_dest = dma_map_page(dev->dev, page, offset, len, DMA_FROM_DEVICE); 5280036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, 0); 5290036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams 5300036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams if (!tx) { 5310036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE); 5320036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_unmap_page(dev->dev, dma_dest, len, DMA_FROM_DEVICE); 5337405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams return -ENOMEM; 5340036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams } 5357405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 5367405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams tx->ack = 1; 5377405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams tx->callback = NULL; 5387405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams cookie = tx->tx_submit(tx); 5397405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 5407405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams cpu = get_cpu(); 5417405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; 5427405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams per_cpu_ptr(chan->local, cpu)->memcpy_count++; 5437405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams put_cpu(); 5447405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 5457405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams return cookie; 5467405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams} 5477405f74badf46b5d023c5d2b670b4471525f6c91Dan WilliamsEXPORT_SYMBOL(dma_async_memcpy_buf_to_pg); 5487405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 5497405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams/** 5507405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * dma_async_memcpy_pg_to_pg - offloaded copy from page to page 5517405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @chan: DMA channel to offload copy to 5527405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @dest_pg: destination page 5537405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @dest_off: offset in page to copy to 5547405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @src_pg: source page 5557405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @src_off: offset in page to copy from 5567405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * @len: length 5577405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * 5587405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * Both @dest_page/@dest_off and @src_page/@src_off must be mappable to a bus 5597405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * address according to the DMA mapping API rules for streaming mappings. 5607405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * Both @dest_page/@dest_off and @src_page/@src_off must stay memory resident 5617405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams * (kernel memory or locked user space pages). 5627405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams */ 5637405f74badf46b5d023c5d2b670b4471525f6c91Dan Williamsdma_cookie_t 5647405f74badf46b5d023c5d2b670b4471525f6c91Dan Williamsdma_async_memcpy_pg_to_pg(struct dma_chan *chan, struct page *dest_pg, 5657405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams unsigned int dest_off, struct page *src_pg, unsigned int src_off, 5667405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams size_t len) 5677405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams{ 5687405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams struct dma_device *dev = chan->device; 5697405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams struct dma_async_tx_descriptor *tx; 5700036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_addr_t dma_dest, dma_src; 5717405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams dma_cookie_t cookie; 5727405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams int cpu; 5737405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 5740036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_src = dma_map_page(dev->dev, src_pg, src_off, len, DMA_TO_DEVICE); 5750036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_dest = dma_map_page(dev->dev, dest_pg, dest_off, len, 5760036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams DMA_FROM_DEVICE); 5770036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, 0); 5780036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams 5790036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams if (!tx) { 5800036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_unmap_page(dev->dev, dma_src, len, DMA_TO_DEVICE); 5810036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_unmap_page(dev->dev, dma_dest, len, DMA_FROM_DEVICE); 5827405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams return -ENOMEM; 5830036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams } 5847405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 5857405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams tx->ack = 1; 5867405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams tx->callback = NULL; 5877405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams cookie = tx->tx_submit(tx); 5887405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 5897405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams cpu = get_cpu(); 5907405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; 5917405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams per_cpu_ptr(chan->local, cpu)->memcpy_count++; 5927405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams put_cpu(); 5937405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 5947405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams return cookie; 5957405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams} 5967405f74badf46b5d023c5d2b670b4471525f6c91Dan WilliamsEXPORT_SYMBOL(dma_async_memcpy_pg_to_pg); 5977405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 5987405f74badf46b5d023c5d2b670b4471525f6c91Dan Williamsvoid dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx, 5997405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams struct dma_chan *chan) 6007405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams{ 6017405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams tx->chan = chan; 6027405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams spin_lock_init(&tx->lock); 6037405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams INIT_LIST_HEAD(&tx->depend_node); 6047405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams INIT_LIST_HEAD(&tx->depend_list); 6057405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams} 6067405f74badf46b5d023c5d2b670b4471525f6c91Dan WilliamsEXPORT_SYMBOL(dma_async_tx_descriptor_init); 6077405f74badf46b5d023c5d2b670b4471525f6c91Dan Williams 608c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leechstatic int __init dma_bus_init(void) 609c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech{ 610c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech mutex_init(&dma_list_mutex); 611c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech return class_register(&dma_devclass); 612c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech} 613c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leechsubsys_initcall(dma_bus_init); 614c13c8260da3155f2cefb63b0d1b7dcdcb405c644Chris Leech 615