1c211092313b90f898dec61f35207fc282d1eadc3Dan Williams/* 2c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * offload engine driver for the Intel Xscale series of i/o processors 3c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * Copyright © 2006, Intel Corporation. 4c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * 5c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * This program is free software; you can redistribute it and/or modify it 6c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * under the terms and conditions of the GNU General Public License, 7c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * version 2, as published by the Free Software Foundation. 8c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * 9c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * This program is distributed in the hope it will be useful, but WITHOUT 10c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * more details. 13c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * 14c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * You should have received a copy of the GNU General Public License along with 15c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * this program; if not, write to the Free Software Foundation, Inc., 16c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 17c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * 18c211092313b90f898dec61f35207fc282d1eadc3Dan Williams */ 19c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 20c211092313b90f898dec61f35207fc282d1eadc3Dan Williams/* 21c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * This driver supports the asynchrounous DMA copy and RAID engines available 22c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) 23c211092313b90f898dec61f35207fc282d1eadc3Dan Williams */ 24c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 25c211092313b90f898dec61f35207fc282d1eadc3Dan Williams#include <linux/init.h> 26c211092313b90f898dec61f35207fc282d1eadc3Dan Williams#include <linux/module.h> 27c211092313b90f898dec61f35207fc282d1eadc3Dan Williams#include <linux/delay.h> 28c211092313b90f898dec61f35207fc282d1eadc3Dan Williams#include <linux/dma-mapping.h> 29c211092313b90f898dec61f35207fc282d1eadc3Dan Williams#include <linux/spinlock.h> 30c211092313b90f898dec61f35207fc282d1eadc3Dan Williams#include <linux/interrupt.h> 31c211092313b90f898dec61f35207fc282d1eadc3Dan Williams#include <linux/platform_device.h> 32c211092313b90f898dec61f35207fc282d1eadc3Dan Williams#include <linux/memory.h> 33c211092313b90f898dec61f35207fc282d1eadc3Dan Williams#include <linux/ioport.h> 34f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams#include <linux/raid/pq.h> 355a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 36c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 37a09e64fbc0094e3073dbb09c3b4bfe4ab669244bRussell King#include <mach/adma.h> 38c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 39d2ebfb335b0426deb1a4fb14e4e926d81ecd8235Russell King - ARM Linux#include "dmaengine.h" 40d2ebfb335b0426deb1a4fb14e4e926d81ecd8235Russell King - ARM Linux 41c211092313b90f898dec61f35207fc282d1eadc3Dan Williams#define to_iop_adma_chan(chan) container_of(chan, struct iop_adma_chan, common) 42c211092313b90f898dec61f35207fc282d1eadc3Dan Williams#define to_iop_adma_device(dev) \ 43c211092313b90f898dec61f35207fc282d1eadc3Dan Williams container_of(dev, struct iop_adma_device, common) 44c211092313b90f898dec61f35207fc282d1eadc3Dan Williams#define tx_to_iop_adma_slot(tx) \ 45c211092313b90f898dec61f35207fc282d1eadc3Dan Williams container_of(tx, struct iop_adma_desc_slot, async_tx) 46c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 47c211092313b90f898dec61f35207fc282d1eadc3Dan Williams/** 48c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * iop_adma_free_slots - flags descriptor slots for reuse 49c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * @slot: Slot to free 50c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * Caller must hold &iop_chan->lock while calling this function 51c211092313b90f898dec61f35207fc282d1eadc3Dan Williams */ 52c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic void iop_adma_free_slots(struct iop_adma_desc_slot *slot) 53c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 54c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int stride = slot->slots_per_op; 55c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 56c211092313b90f898dec61f35207fc282d1eadc3Dan Williams while (stride--) { 57c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot->slots_per_op = 0; 58c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot = list_entry(slot->slot_node.next, 59c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_desc_slot, 60c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot_node); 61c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 62c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 63c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 647bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williamsstatic void 657bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williamsiop_desc_unmap(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc) 667bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams{ 677bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams struct dma_async_tx_descriptor *tx = &desc->async_tx; 687bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams struct iop_adma_desc_slot *unmap = desc->group_head; 697bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams struct device *dev = &iop_chan->device->pdev->dev; 707bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams u32 len = unmap->unmap_len; 717bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams enum dma_ctrl_flags flags = tx->flags; 727bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams u32 src_cnt; 737bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams dma_addr_t addr; 747bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams dma_addr_t dest; 757bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 767bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams src_cnt = unmap->unmap_src_cnt; 777bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams dest = iop_desc_get_dest_addr(unmap, iop_chan); 787bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) { 797bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams enum dma_data_direction dir; 807bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 817bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams if (src_cnt > 1) /* is xor? */ 827bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams dir = DMA_BIDIRECTIONAL; 837bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams else 847bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams dir = DMA_FROM_DEVICE; 857bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 867bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams dma_unmap_page(dev, dest, len, dir); 877bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams } 887bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 897bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) { 907bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams while (src_cnt--) { 917bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams addr = iop_desc_get_src_addr(unmap, iop_chan, src_cnt); 927bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams if (addr == dest) 937bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams continue; 947bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams dma_unmap_page(dev, addr, len, DMA_TO_DEVICE); 957bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams } 967bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams } 977bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams desc->group_head = NULL; 987bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams} 997bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 1007bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williamsstatic void 1017bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williamsiop_desc_unmap_pq(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc) 1027bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams{ 1037bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams struct dma_async_tx_descriptor *tx = &desc->async_tx; 1047bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams struct iop_adma_desc_slot *unmap = desc->group_head; 1057bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams struct device *dev = &iop_chan->device->pdev->dev; 1067bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams u32 len = unmap->unmap_len; 1077bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams enum dma_ctrl_flags flags = tx->flags; 1087bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams u32 src_cnt = unmap->unmap_src_cnt; 1097bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams dma_addr_t pdest = iop_desc_get_dest_addr(unmap, iop_chan); 1107bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams dma_addr_t qdest = iop_desc_get_qdest_addr(unmap, iop_chan); 1117bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams int i; 1127bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 1137bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams if (tx->flags & DMA_PREP_CONTINUE) 1147bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams src_cnt -= 3; 1157bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 1167bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP) && !desc->pq_check_result) { 1177bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams dma_unmap_page(dev, pdest, len, DMA_BIDIRECTIONAL); 1187bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams dma_unmap_page(dev, qdest, len, DMA_BIDIRECTIONAL); 1197bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams } 1207bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 1217bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) { 1227bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams dma_addr_t addr; 1237bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 1247bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams for (i = 0; i < src_cnt; i++) { 1257bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams addr = iop_desc_get_src_addr(unmap, iop_chan, i); 1267bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams dma_unmap_page(dev, addr, len, DMA_TO_DEVICE); 1277bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams } 1287bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams if (desc->pq_check_result) { 1297bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams dma_unmap_page(dev, pdest, len, DMA_TO_DEVICE); 1307bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams dma_unmap_page(dev, qdest, len, DMA_TO_DEVICE); 1317bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams } 1327bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams } 1337bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 1347bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams desc->group_head = NULL; 1357bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams} 1367bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 1377bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 138c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic dma_cookie_t 139c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsiop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc, 140c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_chan *iop_chan, dma_cookie_t cookie) 141c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 142507fbec4cff442ebce6706db34603bfb9cc3b5a9Dan Williams struct dma_async_tx_descriptor *tx = &desc->async_tx; 143507fbec4cff442ebce6706db34603bfb9cc3b5a9Dan Williams 144507fbec4cff442ebce6706db34603bfb9cc3b5a9Dan Williams BUG_ON(tx->cookie < 0); 145507fbec4cff442ebce6706db34603bfb9cc3b5a9Dan Williams if (tx->cookie > 0) { 146507fbec4cff442ebce6706db34603bfb9cc3b5a9Dan Williams cookie = tx->cookie; 147507fbec4cff442ebce6706db34603bfb9cc3b5a9Dan Williams tx->cookie = 0; 148c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 149c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* call the callback (must not sleep or submit new 150c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * operations to this channel) 151c211092313b90f898dec61f35207fc282d1eadc3Dan Williams */ 152507fbec4cff442ebce6706db34603bfb9cc3b5a9Dan Williams if (tx->callback) 153507fbec4cff442ebce6706db34603bfb9cc3b5a9Dan Williams tx->callback(tx->callback_param); 154c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 155c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* unmap dma addresses 156c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * (unmap_single vs unmap_page?) 157c211092313b90f898dec61f35207fc282d1eadc3Dan Williams */ 158c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (desc->group_head && desc->unmap_len) { 1597bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams if (iop_desc_is_pq(desc)) 1607bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams iop_desc_unmap_pq(iop_chan, desc); 1617bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams else 1627bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams iop_desc_unmap(iop_chan, desc); 163c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 164c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 165c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 166c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* run dependent operations */ 167507fbec4cff442ebce6706db34603bfb9cc3b5a9Dan Williams dma_run_dependencies(tx); 168c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 169c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return cookie; 170c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 171c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 172c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic int 173c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsiop_adma_clean_slot(struct iop_adma_desc_slot *desc, 174c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_chan *iop_chan) 175c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 176c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* the client is allowed to attach dependent operations 177c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * until 'ack' is set 178c211092313b90f898dec61f35207fc282d1eadc3Dan Williams */ 179636bdeaa1243327501edfd2a597ed7443eb4239aDan Williams if (!async_tx_test_ack(&desc->async_tx)) 180c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return 0; 181c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 182c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* leave the last descriptor in the chain 183c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * so we can append to it 184c211092313b90f898dec61f35207fc282d1eadc3Dan Williams */ 185c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (desc->chain_node.next == &iop_chan->chain) 186c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return 1; 187c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 188c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_dbg(iop_chan->device->common.dev, 189c211092313b90f898dec61f35207fc282d1eadc3Dan Williams "\tfree slot: %d slots_per_op: %d\n", 190c211092313b90f898dec61f35207fc282d1eadc3Dan Williams desc->idx, desc->slots_per_op); 191c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 192c211092313b90f898dec61f35207fc282d1eadc3Dan Williams list_del(&desc->chain_node); 193c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_adma_free_slots(desc); 194c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 195c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return 0; 196c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 197c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 198c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic void __iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan) 199c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 200c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_desc_slot *iter, *_iter, *grp_start = NULL; 201c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_cookie_t cookie = 0; 202c211092313b90f898dec61f35207fc282d1eadc3Dan Williams u32 current_desc = iop_chan_get_current_descriptor(iop_chan); 203c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int busy = iop_chan_is_busy(iop_chan); 204c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int seen_current = 0, slot_cnt = 0, slots_per_op = 0; 205c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 2063d9b525b69bc3302d8355e5f5cf081a856c211e0Harvey Harrison dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); 207c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* free completed slots from the chain starting with 208c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * the oldest descriptor 209c211092313b90f898dec61f35207fc282d1eadc3Dan Williams */ 210c211092313b90f898dec61f35207fc282d1eadc3Dan Williams list_for_each_entry_safe(iter, _iter, &iop_chan->chain, 211c211092313b90f898dec61f35207fc282d1eadc3Dan Williams chain_node) { 212c211092313b90f898dec61f35207fc282d1eadc3Dan Williams pr_debug("\tcookie: %d slot: %d busy: %d " 213c211092313b90f898dec61f35207fc282d1eadc3Dan Williams "this_desc: %#x next_desc: %#x ack: %d\n", 214c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iter->async_tx.cookie, iter->idx, busy, 215c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iter->async_tx.phys, iop_desc_get_next_desc(iter), 216636bdeaa1243327501edfd2a597ed7443eb4239aDan Williams async_tx_test_ack(&iter->async_tx)); 217c211092313b90f898dec61f35207fc282d1eadc3Dan Williams prefetch(_iter); 218c211092313b90f898dec61f35207fc282d1eadc3Dan Williams prefetch(&_iter->async_tx); 219c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 220c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* do not advance past the current descriptor loaded into the 221c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * hardware channel, subsequent descriptors are either in 222c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * process or have not been submitted 223c211092313b90f898dec61f35207fc282d1eadc3Dan Williams */ 224c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (seen_current) 225c211092313b90f898dec61f35207fc282d1eadc3Dan Williams break; 226c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 227c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* stop the search if we reach the current descriptor and the 228c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * channel is busy, or if it appears that the current descriptor 229c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * needs to be re-read (i.e. has been appended to) 230c211092313b90f898dec61f35207fc282d1eadc3Dan Williams */ 231c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (iter->async_tx.phys == current_desc) { 232c211092313b90f898dec61f35207fc282d1eadc3Dan Williams BUG_ON(seen_current++); 233c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (busy || iop_desc_get_next_desc(iter)) 234c211092313b90f898dec61f35207fc282d1eadc3Dan Williams break; 235c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 236c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 237c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* detect the start of a group transaction */ 238c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (!slot_cnt && !slots_per_op) { 239c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot_cnt = iter->slot_cnt; 240c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slots_per_op = iter->slots_per_op; 241c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (slot_cnt <= slots_per_op) { 242c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot_cnt = 0; 243c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slots_per_op = 0; 244c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 245c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 246c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 247c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (slot_cnt) { 248c211092313b90f898dec61f35207fc282d1eadc3Dan Williams pr_debug("\tgroup++\n"); 249c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (!grp_start) 250c211092313b90f898dec61f35207fc282d1eadc3Dan Williams grp_start = iter; 251c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot_cnt -= slots_per_op; 252c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 253c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 254c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* all the members of a group are complete */ 255c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (slots_per_op != 0 && slot_cnt == 0) { 256c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_desc_slot *grp_iter, *_grp_iter; 257c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int end_of_chain = 0; 258c211092313b90f898dec61f35207fc282d1eadc3Dan Williams pr_debug("\tgroup end\n"); 259c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 260c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* collect the total results */ 261c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (grp_start->xor_check_result) { 262c211092313b90f898dec61f35207fc282d1eadc3Dan Williams u32 zero_sum_result = 0; 263c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot_cnt = grp_start->slot_cnt; 264c211092313b90f898dec61f35207fc282d1eadc3Dan Williams grp_iter = grp_start; 265c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 266c211092313b90f898dec61f35207fc282d1eadc3Dan Williams list_for_each_entry_from(grp_iter, 267c211092313b90f898dec61f35207fc282d1eadc3Dan Williams &iop_chan->chain, chain_node) { 268c211092313b90f898dec61f35207fc282d1eadc3Dan Williams zero_sum_result |= 269c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_desc_get_zero_result(grp_iter); 270c211092313b90f898dec61f35207fc282d1eadc3Dan Williams pr_debug("\titer%d result: %d\n", 271c211092313b90f898dec61f35207fc282d1eadc3Dan Williams grp_iter->idx, zero_sum_result); 272c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot_cnt -= slots_per_op; 273c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (slot_cnt == 0) 274c211092313b90f898dec61f35207fc282d1eadc3Dan Williams break; 275c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 276c211092313b90f898dec61f35207fc282d1eadc3Dan Williams pr_debug("\tgrp_start->xor_check_result: %p\n", 277c211092313b90f898dec61f35207fc282d1eadc3Dan Williams grp_start->xor_check_result); 278c211092313b90f898dec61f35207fc282d1eadc3Dan Williams *grp_start->xor_check_result = zero_sum_result; 279c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 280c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 281c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* clean up the group */ 282c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot_cnt = grp_start->slot_cnt; 283c211092313b90f898dec61f35207fc282d1eadc3Dan Williams grp_iter = grp_start; 284c211092313b90f898dec61f35207fc282d1eadc3Dan Williams list_for_each_entry_safe_from(grp_iter, _grp_iter, 285c211092313b90f898dec61f35207fc282d1eadc3Dan Williams &iop_chan->chain, chain_node) { 286c211092313b90f898dec61f35207fc282d1eadc3Dan Williams cookie = iop_adma_run_tx_complete_actions( 287c211092313b90f898dec61f35207fc282d1eadc3Dan Williams grp_iter, iop_chan, cookie); 288c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 289c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot_cnt -= slots_per_op; 290c211092313b90f898dec61f35207fc282d1eadc3Dan Williams end_of_chain = iop_adma_clean_slot(grp_iter, 291c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan); 292c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 293c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (slot_cnt == 0 || end_of_chain) 294c211092313b90f898dec61f35207fc282d1eadc3Dan Williams break; 295c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 296c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 297c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* the group should be complete at this point */ 298c211092313b90f898dec61f35207fc282d1eadc3Dan Williams BUG_ON(slot_cnt); 299c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 300c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slots_per_op = 0; 301c211092313b90f898dec61f35207fc282d1eadc3Dan Williams grp_start = NULL; 302c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (end_of_chain) 303c211092313b90f898dec61f35207fc282d1eadc3Dan Williams break; 304c211092313b90f898dec61f35207fc282d1eadc3Dan Williams else 305c211092313b90f898dec61f35207fc282d1eadc3Dan Williams continue; 306c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } else if (slots_per_op) /* wait for group completion */ 307c211092313b90f898dec61f35207fc282d1eadc3Dan Williams continue; 308c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 309c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* write back zero sum results (single descriptor case) */ 310c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (iter->xor_check_result && iter->async_tx.cookie) 311c211092313b90f898dec61f35207fc282d1eadc3Dan Williams *iter->xor_check_result = 312c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_desc_get_zero_result(iter); 313c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 314c211092313b90f898dec61f35207fc282d1eadc3Dan Williams cookie = iop_adma_run_tx_complete_actions( 315c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iter, iop_chan, cookie); 316c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 317c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (iop_adma_clean_slot(iter, iop_chan)) 318c211092313b90f898dec61f35207fc282d1eadc3Dan Williams break; 319c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 320c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 321c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (cookie > 0) { 3224d4e58de32a192fea65ab84509d17d199bd291c8Russell King - ARM Linux iop_chan->common.completed_cookie = cookie; 323c211092313b90f898dec61f35207fc282d1eadc3Dan Williams pr_debug("\tcompleted cookie %d\n", cookie); 324c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 325c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 326c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 327c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic void 328c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsiop_adma_slot_cleanup(struct iop_adma_chan *iop_chan) 329c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 330c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_lock_bh(&iop_chan->lock); 331c211092313b90f898dec61f35207fc282d1eadc3Dan Williams __iop_adma_slot_cleanup(iop_chan); 332c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_unlock_bh(&iop_chan->lock); 333c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 334c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 335c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic void iop_adma_tasklet(unsigned long data) 336c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 33719242d7233df7d658405d4b7ee1758d21414cfaaDan Williams struct iop_adma_chan *iop_chan = (struct iop_adma_chan *) data; 33819242d7233df7d658405d4b7ee1758d21414cfaaDan Williams 33972be12f0c39df46832403cbfd82e132a883f5ddcDan Williams /* lockdep will flag depedency submissions as potentially 34072be12f0c39df46832403cbfd82e132a883f5ddcDan Williams * recursive locking, this is not the case as a dependency 34172be12f0c39df46832403cbfd82e132a883f5ddcDan Williams * submission will never recurse a channels submit routine. 34272be12f0c39df46832403cbfd82e132a883f5ddcDan Williams * There are checks in async_tx.c to prevent this. 34372be12f0c39df46832403cbfd82e132a883f5ddcDan Williams */ 34472be12f0c39df46832403cbfd82e132a883f5ddcDan Williams spin_lock_nested(&iop_chan->lock, SINGLE_DEPTH_NESTING); 34519242d7233df7d658405d4b7ee1758d21414cfaaDan Williams __iop_adma_slot_cleanup(iop_chan); 34619242d7233df7d658405d4b7ee1758d21414cfaaDan Williams spin_unlock(&iop_chan->lock); 347c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 348c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 349c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic struct iop_adma_desc_slot * 350c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsiop_adma_alloc_slots(struct iop_adma_chan *iop_chan, int num_slots, 351c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int slots_per_op) 352c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 353c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_desc_slot *iter, *_iter, *alloc_start = NULL; 354e73ef9acfd30f36bf7c60237ecffe7bbca8068d6Denis Cheng LIST_HEAD(chain); 355c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int slots_found, retry = 0; 356c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 357c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* start search from the last allocated descrtiptor 358c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * if a contiguous allocation can not be found start searching 359c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * from the beginning of the list 360c211092313b90f898dec61f35207fc282d1eadc3Dan Williams */ 361c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsretry: 362c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slots_found = 0; 363c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (retry == 0) 364c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iter = iop_chan->last_used; 365c211092313b90f898dec61f35207fc282d1eadc3Dan Williams else 366c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iter = list_entry(&iop_chan->all_slots, 367c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_desc_slot, 368c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot_node); 369c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 370c211092313b90f898dec61f35207fc282d1eadc3Dan Williams list_for_each_entry_safe_continue( 371c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iter, _iter, &iop_chan->all_slots, slot_node) { 372c211092313b90f898dec61f35207fc282d1eadc3Dan Williams prefetch(_iter); 373c211092313b90f898dec61f35207fc282d1eadc3Dan Williams prefetch(&_iter->async_tx); 374c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (iter->slots_per_op) { 375c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* give up after finding the first busy slot 376c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * on the second pass through the list 377c211092313b90f898dec61f35207fc282d1eadc3Dan Williams */ 378c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (retry) 379c211092313b90f898dec61f35207fc282d1eadc3Dan Williams break; 380c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 381c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slots_found = 0; 382c211092313b90f898dec61f35207fc282d1eadc3Dan Williams continue; 383c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 384c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 385c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* start the allocation if the slot is correctly aligned */ 386c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (!slots_found++) { 387c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (iop_desc_is_aligned(iter, slots_per_op)) 388c211092313b90f898dec61f35207fc282d1eadc3Dan Williams alloc_start = iter; 389c211092313b90f898dec61f35207fc282d1eadc3Dan Williams else { 390c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slots_found = 0; 391c211092313b90f898dec61f35207fc282d1eadc3Dan Williams continue; 392c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 393c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 394c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 395c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (slots_found == num_slots) { 396c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_desc_slot *alloc_tail = NULL; 397c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_desc_slot *last_used = NULL; 398c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iter = alloc_start; 399c211092313b90f898dec61f35207fc282d1eadc3Dan Williams while (num_slots) { 400c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int i; 401c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_dbg(iop_chan->device->common.dev, 402c211092313b90f898dec61f35207fc282d1eadc3Dan Williams "allocated slot: %d " 403c211092313b90f898dec61f35207fc282d1eadc3Dan Williams "(desc %p phys: %#x) slots_per_op %d\n", 404c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iter->idx, iter->hw_desc, 405c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iter->async_tx.phys, slots_per_op); 406c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 407c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* pre-ack all but the last descriptor */ 408c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (num_slots != slots_per_op) 409636bdeaa1243327501edfd2a597ed7443eb4239aDan Williams async_tx_ack(&iter->async_tx); 410c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 411c211092313b90f898dec61f35207fc282d1eadc3Dan Williams list_add_tail(&iter->chain_node, &chain); 412c211092313b90f898dec61f35207fc282d1eadc3Dan Williams alloc_tail = iter; 413c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iter->async_tx.cookie = 0; 414c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iter->slot_cnt = num_slots; 415c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iter->xor_check_result = NULL; 416c211092313b90f898dec61f35207fc282d1eadc3Dan Williams for (i = 0; i < slots_per_op; i++) { 417c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iter->slots_per_op = slots_per_op - i; 418c211092313b90f898dec61f35207fc282d1eadc3Dan Williams last_used = iter; 419c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iter = list_entry(iter->slot_node.next, 420c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_desc_slot, 421c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot_node); 422c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 423c211092313b90f898dec61f35207fc282d1eadc3Dan Williams num_slots -= slots_per_op; 424c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 425c211092313b90f898dec61f35207fc282d1eadc3Dan Williams alloc_tail->group_head = alloc_start; 426c211092313b90f898dec61f35207fc282d1eadc3Dan Williams alloc_tail->async_tx.cookie = -EBUSY; 427308136d1abcb2d759bac40ed4f5d42ac4af59d8bDan Williams list_splice(&chain, &alloc_tail->tx_list); 428c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan->last_used = last_used; 429c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_desc_clear_next_desc(alloc_start); 430c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_desc_clear_next_desc(alloc_tail); 431c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return alloc_tail; 432c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 433c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 434c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (!retry++) 435c211092313b90f898dec61f35207fc282d1eadc3Dan Williams goto retry; 436c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 437c7141d005a19d2a0a316b3bf9c170d3bedf07bfdDan Williams /* perform direct reclaim if the allocation fails */ 438c7141d005a19d2a0a316b3bf9c170d3bedf07bfdDan Williams __iop_adma_slot_cleanup(iop_chan); 439c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 440c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return NULL; 441c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 442c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 443c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic void iop_adma_check_threshold(struct iop_adma_chan *iop_chan) 444c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 445c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_dbg(iop_chan->device->common.dev, "pending: %d\n", 446c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan->pending); 447c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 448c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (iop_chan->pending >= IOP_ADMA_THRESHOLD) { 449c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan->pending = 0; 450c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan_append(iop_chan); 451c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 452c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 453c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 454c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic dma_cookie_t 455c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsiop_adma_tx_submit(struct dma_async_tx_descriptor *tx) 456c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 457c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx); 458c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_chan *iop_chan = to_iop_adma_chan(tx->chan); 459c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_desc_slot *grp_start, *old_chain_tail; 460c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int slot_cnt; 461c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int slots_per_op; 462c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_cookie_t cookie; 463137cb55c6dcd56cb367285adaf15f808a2a9fec7Dan Williams dma_addr_t next_dma; 464c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 465c211092313b90f898dec61f35207fc282d1eadc3Dan Williams grp_start = sw_desc->group_head; 466c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot_cnt = grp_start->slot_cnt; 467c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slots_per_op = grp_start->slots_per_op; 468c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 469c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_lock_bh(&iop_chan->lock); 470884485e1f12dcd39390f042e772cdbefc9ebb750Russell King - ARM Linux cookie = dma_cookie_assign(tx); 471c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 472c211092313b90f898dec61f35207fc282d1eadc3Dan Williams old_chain_tail = list_entry(iop_chan->chain.prev, 473c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_desc_slot, chain_node); 474308136d1abcb2d759bac40ed4f5d42ac4af59d8bDan Williams list_splice_init(&sw_desc->tx_list, 475c211092313b90f898dec61f35207fc282d1eadc3Dan Williams &old_chain_tail->chain_node); 476c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 477c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* fix up the hardware chain */ 478137cb55c6dcd56cb367285adaf15f808a2a9fec7Dan Williams next_dma = grp_start->async_tx.phys; 479137cb55c6dcd56cb367285adaf15f808a2a9fec7Dan Williams iop_desc_set_next_desc(old_chain_tail, next_dma); 480137cb55c6dcd56cb367285adaf15f808a2a9fec7Dan Williams BUG_ON(iop_desc_get_next_desc(old_chain_tail) != next_dma); /* flush */ 481c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 482137cb55c6dcd56cb367285adaf15f808a2a9fec7Dan Williams /* check for pre-chained descriptors */ 48365e503814dec83c7b2ac955e75919d009109c919Dan Williams iop_paranoia(iop_desc_get_next_desc(sw_desc)); 484c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 485c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* increment the pending count by the number of slots 486c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * memcpy operations have a 1:1 (slot:operation) relation 487c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * other operations are heavier and will pop the threshold 488c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * more often. 489c211092313b90f898dec61f35207fc282d1eadc3Dan Williams */ 490c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan->pending += slot_cnt; 491c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_adma_check_threshold(iop_chan); 492c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_unlock_bh(&iop_chan->lock); 493c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 494c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_dbg(iop_chan->device->common.dev, "%s cookie: %d slot: %d\n", 4953d9b525b69bc3302d8355e5f5cf081a856c211e0Harvey Harrison __func__, sw_desc->async_tx.cookie, sw_desc->idx); 496c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 497c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return cookie; 498c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 499c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 500c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan); 501c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan); 502c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 5035eb907aaaf7a316a0097ff9f8c21bf9fc468a1f1Dan Williams/** 5045eb907aaaf7a316a0097ff9f8c21bf9fc468a1f1Dan Williams * iop_adma_alloc_chan_resources - returns the number of allocated descriptors 5055eb907aaaf7a316a0097ff9f8c21bf9fc468a1f1Dan Williams * @chan - allocate descriptor resources for this channel 5065eb907aaaf7a316a0097ff9f8c21bf9fc468a1f1Dan Williams * @client - current client requesting the channel be ready for requests 5075eb907aaaf7a316a0097ff9f8c21bf9fc468a1f1Dan Williams * 5085eb907aaaf7a316a0097ff9f8c21bf9fc468a1f1Dan Williams * Note: We keep the slots for 1 operation on iop_chan->chain at all times. To 5095eb907aaaf7a316a0097ff9f8c21bf9fc468a1f1Dan Williams * avoid deadlock, via async_xor, num_descs_in_pool must at a minimum be 5105eb907aaaf7a316a0097ff9f8c21bf9fc468a1f1Dan Williams * greater than 2x the number slots needed to satisfy a device->max_xor 5115eb907aaaf7a316a0097ff9f8c21bf9fc468a1f1Dan Williams * request. 5125eb907aaaf7a316a0097ff9f8c21bf9fc468a1f1Dan Williams * */ 513aa1e6f1a385eb2b04171ec841f3b760091e4a8eeDan Williamsstatic int iop_adma_alloc_chan_resources(struct dma_chan *chan) 514c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 515c211092313b90f898dec61f35207fc282d1eadc3Dan Williams char *hw_desc; 516c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int idx; 517c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 518c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_desc_slot *slot = NULL; 519c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int init = iop_chan->slots_allocated ? 0 : 1; 520c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_platform_data *plat_data = 521c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan->device->pdev->dev.platform_data; 522c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int num_descs_in_pool = plat_data->pool_size/IOP_ADMA_SLOT_SIZE; 523c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 524c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* Allocate descriptor slots */ 525c211092313b90f898dec61f35207fc282d1eadc3Dan Williams do { 526c211092313b90f898dec61f35207fc282d1eadc3Dan Williams idx = iop_chan->slots_allocated; 527c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (idx == num_descs_in_pool) 528c211092313b90f898dec61f35207fc282d1eadc3Dan Williams break; 529c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 530c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot = kzalloc(sizeof(*slot), GFP_KERNEL); 531c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (!slot) { 532c211092313b90f898dec61f35207fc282d1eadc3Dan Williams printk(KERN_INFO "IOP ADMA Channel only initialized" 533c211092313b90f898dec61f35207fc282d1eadc3Dan Williams " %d descriptor slots", idx); 534c211092313b90f898dec61f35207fc282d1eadc3Dan Williams break; 535c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 536c211092313b90f898dec61f35207fc282d1eadc3Dan Williams hw_desc = (char *) iop_chan->device->dma_desc_pool_virt; 537c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot->hw_desc = (void *) &hw_desc[idx * IOP_ADMA_SLOT_SIZE]; 538c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 539c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_async_tx_descriptor_init(&slot->async_tx, chan); 540c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot->async_tx.tx_submit = iop_adma_tx_submit; 541308136d1abcb2d759bac40ed4f5d42ac4af59d8bDan Williams INIT_LIST_HEAD(&slot->tx_list); 542c211092313b90f898dec61f35207fc282d1eadc3Dan Williams INIT_LIST_HEAD(&slot->chain_node); 543c211092313b90f898dec61f35207fc282d1eadc3Dan Williams INIT_LIST_HEAD(&slot->slot_node); 544c211092313b90f898dec61f35207fc282d1eadc3Dan Williams hw_desc = (char *) iop_chan->device->dma_desc_pool; 545c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot->async_tx.phys = 546c211092313b90f898dec61f35207fc282d1eadc3Dan Williams (dma_addr_t) &hw_desc[idx * IOP_ADMA_SLOT_SIZE]; 547c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot->idx = idx; 548c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 549c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_lock_bh(&iop_chan->lock); 550c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan->slots_allocated++; 551c211092313b90f898dec61f35207fc282d1eadc3Dan Williams list_add_tail(&slot->slot_node, &iop_chan->all_slots); 552c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_unlock_bh(&iop_chan->lock); 553c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } while (iop_chan->slots_allocated < num_descs_in_pool); 554c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 555c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (idx && !iop_chan->last_used) 556c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan->last_used = list_entry(iop_chan->all_slots.next, 557c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_desc_slot, 558c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot_node); 559c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 560c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_dbg(iop_chan->device->common.dev, 561c211092313b90f898dec61f35207fc282d1eadc3Dan Williams "allocated %d descriptor slots last_used: %p\n", 562c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan->slots_allocated, iop_chan->last_used); 563c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 564c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* initialize the channel and the chain with a null operation */ 565c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (init) { 566c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (dma_has_cap(DMA_MEMCPY, 567c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan->device->common.cap_mask)) 568c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan_start_null_memcpy(iop_chan); 569c211092313b90f898dec61f35207fc282d1eadc3Dan Williams else if (dma_has_cap(DMA_XOR, 570c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan->device->common.cap_mask)) 571c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan_start_null_xor(iop_chan); 572c211092313b90f898dec61f35207fc282d1eadc3Dan Williams else 573c211092313b90f898dec61f35207fc282d1eadc3Dan Williams BUG(); 574c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 575c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 576c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return (idx > 0) ? idx : -ENOMEM; 577c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 578c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 579c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic struct dma_async_tx_descriptor * 580636bdeaa1243327501edfd2a597ed7443eb4239aDan Williamsiop_adma_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags) 581c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 582c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 583c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_desc_slot *sw_desc, *grp_start; 584c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int slot_cnt, slots_per_op; 585c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 5863d9b525b69bc3302d8355e5f5cf081a856c211e0Harvey Harrison dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); 587c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 588c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_lock_bh(&iop_chan->lock); 589c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot_cnt = iop_chan_interrupt_slot_count(&slots_per_op, iop_chan); 590c211092313b90f898dec61f35207fc282d1eadc3Dan Williams sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 591c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (sw_desc) { 592c211092313b90f898dec61f35207fc282d1eadc3Dan Williams grp_start = sw_desc->group_head; 593c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_desc_init_interrupt(grp_start, iop_chan); 594c211092313b90f898dec61f35207fc282d1eadc3Dan Williams grp_start->unmap_len = 0; 595636bdeaa1243327501edfd2a597ed7443eb4239aDan Williams sw_desc->async_tx.flags = flags; 596c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 597c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_unlock_bh(&iop_chan->lock); 598c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 599c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return sw_desc ? &sw_desc->async_tx : NULL; 600c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 601c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 602c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic struct dma_async_tx_descriptor * 6030036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williamsiop_adma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest, 604d4c56f97ff21df405d0cebe11f49e3c3c79662b5Dan Williams dma_addr_t dma_src, size_t len, unsigned long flags) 605c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 606c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 607c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_desc_slot *sw_desc, *grp_start; 608c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int slot_cnt, slots_per_op; 609c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 610c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (unlikely(!len)) 611c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return NULL; 612e2ec771a99a5cf231c9dea4da26238bf073e1e9cColy Li BUG_ON(len > IOP_ADMA_MAX_BYTE_COUNT); 613c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 614c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_dbg(iop_chan->device->common.dev, "%s len: %u\n", 6153d9b525b69bc3302d8355e5f5cf081a856c211e0Harvey Harrison __func__, len); 616c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 617c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_lock_bh(&iop_chan->lock); 618c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot_cnt = iop_chan_memcpy_slot_count(len, &slots_per_op); 619c211092313b90f898dec61f35207fc282d1eadc3Dan Williams sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 620c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (sw_desc) { 621c211092313b90f898dec61f35207fc282d1eadc3Dan Williams grp_start = sw_desc->group_head; 622d4c56f97ff21df405d0cebe11f49e3c3c79662b5Dan Williams iop_desc_init_memcpy(grp_start, flags); 623c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_desc_set_byte_count(grp_start, iop_chan, len); 6240036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); 6250036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams iop_desc_set_memcpy_src_addr(grp_start, dma_src); 626c211092313b90f898dec61f35207fc282d1eadc3Dan Williams sw_desc->unmap_src_cnt = 1; 627c211092313b90f898dec61f35207fc282d1eadc3Dan Williams sw_desc->unmap_len = len; 628636bdeaa1243327501edfd2a597ed7443eb4239aDan Williams sw_desc->async_tx.flags = flags; 629c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 630c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_unlock_bh(&iop_chan->lock); 631c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 632c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return sw_desc ? &sw_desc->async_tx : NULL; 633c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 634c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 635c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic struct dma_async_tx_descriptor * 6360036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williamsiop_adma_prep_dma_memset(struct dma_chan *chan, dma_addr_t dma_dest, 637d4c56f97ff21df405d0cebe11f49e3c3c79662b5Dan Williams int value, size_t len, unsigned long flags) 638c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 639c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 640c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_desc_slot *sw_desc, *grp_start; 641c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int slot_cnt, slots_per_op; 642c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 643c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (unlikely(!len)) 644c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return NULL; 645e2ec771a99a5cf231c9dea4da26238bf073e1e9cColy Li BUG_ON(len > IOP_ADMA_MAX_BYTE_COUNT); 646c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 647c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_dbg(iop_chan->device->common.dev, "%s len: %u\n", 6483d9b525b69bc3302d8355e5f5cf081a856c211e0Harvey Harrison __func__, len); 649c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 650c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_lock_bh(&iop_chan->lock); 651c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot_cnt = iop_chan_memset_slot_count(len, &slots_per_op); 652c211092313b90f898dec61f35207fc282d1eadc3Dan Williams sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 653c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (sw_desc) { 654c211092313b90f898dec61f35207fc282d1eadc3Dan Williams grp_start = sw_desc->group_head; 655d4c56f97ff21df405d0cebe11f49e3c3c79662b5Dan Williams iop_desc_init_memset(grp_start, flags); 656c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_desc_set_byte_count(grp_start, iop_chan, len); 657c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_desc_set_block_fill_val(grp_start, value); 6580036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); 659c211092313b90f898dec61f35207fc282d1eadc3Dan Williams sw_desc->unmap_src_cnt = 1; 660c211092313b90f898dec61f35207fc282d1eadc3Dan Williams sw_desc->unmap_len = len; 661636bdeaa1243327501edfd2a597ed7443eb4239aDan Williams sw_desc->async_tx.flags = flags; 662c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 663c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_unlock_bh(&iop_chan->lock); 664c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 665c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return sw_desc ? &sw_desc->async_tx : NULL; 666c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 667c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 668c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic struct dma_async_tx_descriptor * 6690036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williamsiop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest, 6700036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_addr_t *dma_src, unsigned int src_cnt, size_t len, 671d4c56f97ff21df405d0cebe11f49e3c3c79662b5Dan Williams unsigned long flags) 672c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 673c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 674c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_desc_slot *sw_desc, *grp_start; 675c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int slot_cnt, slots_per_op; 676c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 677c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (unlikely(!len)) 678c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return NULL; 679e2ec771a99a5cf231c9dea4da26238bf073e1e9cColy Li BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); 680c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 681c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_dbg(iop_chan->device->common.dev, 682d4c56f97ff21df405d0cebe11f49e3c3c79662b5Dan Williams "%s src_cnt: %d len: %u flags: %lx\n", 6833d9b525b69bc3302d8355e5f5cf081a856c211e0Harvey Harrison __func__, src_cnt, len, flags); 684c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 685c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_lock_bh(&iop_chan->lock); 686c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot_cnt = iop_chan_xor_slot_count(len, src_cnt, &slots_per_op); 687c211092313b90f898dec61f35207fc282d1eadc3Dan Williams sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 688c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (sw_desc) { 689c211092313b90f898dec61f35207fc282d1eadc3Dan Williams grp_start = sw_desc->group_head; 690d4c56f97ff21df405d0cebe11f49e3c3c79662b5Dan Williams iop_desc_init_xor(grp_start, src_cnt, flags); 691c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_desc_set_byte_count(grp_start, iop_chan, len); 6920036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); 693c211092313b90f898dec61f35207fc282d1eadc3Dan Williams sw_desc->unmap_src_cnt = src_cnt; 694c211092313b90f898dec61f35207fc282d1eadc3Dan Williams sw_desc->unmap_len = len; 695636bdeaa1243327501edfd2a597ed7443eb4239aDan Williams sw_desc->async_tx.flags = flags; 6960036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams while (src_cnt--) 6970036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams iop_desc_set_xor_src_addr(grp_start, src_cnt, 6980036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_src[src_cnt]); 699c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 700c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_unlock_bh(&iop_chan->lock); 701c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 702c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return sw_desc ? &sw_desc->async_tx : NULL; 703c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 704c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 705c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic struct dma_async_tx_descriptor * 706099f53cb50e45ef617a9f1d63ceec799e489418bDan Williamsiop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src, 707099f53cb50e45ef617a9f1d63ceec799e489418bDan Williams unsigned int src_cnt, size_t len, u32 *result, 708099f53cb50e45ef617a9f1d63ceec799e489418bDan Williams unsigned long flags) 709c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 710c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 711c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_desc_slot *sw_desc, *grp_start; 712c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int slot_cnt, slots_per_op; 713c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 714c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (unlikely(!len)) 715c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return NULL; 716c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 717c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n", 7183d9b525b69bc3302d8355e5f5cf081a856c211e0Harvey Harrison __func__, src_cnt, len); 719c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 720c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_lock_bh(&iop_chan->lock); 721c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot_cnt = iop_chan_zero_sum_slot_count(len, src_cnt, &slots_per_op); 722c211092313b90f898dec61f35207fc282d1eadc3Dan Williams sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 723c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (sw_desc) { 724c211092313b90f898dec61f35207fc282d1eadc3Dan Williams grp_start = sw_desc->group_head; 725d4c56f97ff21df405d0cebe11f49e3c3c79662b5Dan Williams iop_desc_init_zero_sum(grp_start, src_cnt, flags); 726c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_desc_set_zero_sum_byte_count(grp_start, len); 727c211092313b90f898dec61f35207fc282d1eadc3Dan Williams grp_start->xor_check_result = result; 728c211092313b90f898dec61f35207fc282d1eadc3Dan Williams pr_debug("\t%s: grp_start->xor_check_result: %p\n", 7293d9b525b69bc3302d8355e5f5cf081a856c211e0Harvey Harrison __func__, grp_start->xor_check_result); 730c211092313b90f898dec61f35207fc282d1eadc3Dan Williams sw_desc->unmap_src_cnt = src_cnt; 731c211092313b90f898dec61f35207fc282d1eadc3Dan Williams sw_desc->unmap_len = len; 732636bdeaa1243327501edfd2a597ed7443eb4239aDan Williams sw_desc->async_tx.flags = flags; 7330036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams while (src_cnt--) 7340036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams iop_desc_set_zero_sum_src_addr(grp_start, src_cnt, 7350036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_src[src_cnt]); 736c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 737c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_unlock_bh(&iop_chan->lock); 738c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 739c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return sw_desc ? &sw_desc->async_tx : NULL; 740c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 741c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 7427bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williamsstatic struct dma_async_tx_descriptor * 7437bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williamsiop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, 7447bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams unsigned int src_cnt, const unsigned char *scf, size_t len, 7457bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams unsigned long flags) 7467bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams{ 7477bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 7487bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams struct iop_adma_desc_slot *sw_desc, *g; 7497bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams int slot_cnt, slots_per_op; 7507bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams int continue_srcs; 7517bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 7527bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams if (unlikely(!len)) 7537bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams return NULL; 7547bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); 7557bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 7567bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams dev_dbg(iop_chan->device->common.dev, 7577bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams "%s src_cnt: %d len: %u flags: %lx\n", 7587bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams __func__, src_cnt, len, flags); 7597bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 7607bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams if (dmaf_p_disabled_continue(flags)) 7617bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams continue_srcs = 1+src_cnt; 7627bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams else if (dmaf_continue(flags)) 7637bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams continue_srcs = 3+src_cnt; 7647bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams else 7657bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams continue_srcs = 0+src_cnt; 7667bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 7677bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams spin_lock_bh(&iop_chan->lock); 7687bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams slot_cnt = iop_chan_pq_slot_count(len, continue_srcs, &slots_per_op); 7697bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 7707bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams if (sw_desc) { 7717bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams int i; 7727bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 7737bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams g = sw_desc->group_head; 7747bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams iop_desc_set_byte_count(g, iop_chan, len); 7757bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 7767bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams /* even if P is disabled its destination address (bits 7777bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams * [3:0]) must match Q. It is ok if P points to an 7787bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams * invalid address, it won't be written. 7797bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams */ 7807bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams if (flags & DMA_PREP_PQ_DISABLE_P) 7817bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams dst[0] = dst[1] & 0x7; 7827bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 7837bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams iop_desc_set_pq_addr(g, dst); 7847bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams sw_desc->unmap_src_cnt = src_cnt; 7857bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams sw_desc->unmap_len = len; 7867bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams sw_desc->async_tx.flags = flags; 7877bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams for (i = 0; i < src_cnt; i++) 7887bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams iop_desc_set_pq_src_addr(g, i, src[i], scf[i]); 7897bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 7907bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams /* if we are continuing a previous operation factor in 7917bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams * the old p and q values, see the comment for dma_maxpq 7927bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams * in include/linux/dmaengine.h 7937bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams */ 7947bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams if (dmaf_p_disabled_continue(flags)) 7957bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams iop_desc_set_pq_src_addr(g, i++, dst[1], 1); 7967bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams else if (dmaf_continue(flags)) { 7977bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams iop_desc_set_pq_src_addr(g, i++, dst[0], 0); 7987bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams iop_desc_set_pq_src_addr(g, i++, dst[1], 1); 7997bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams iop_desc_set_pq_src_addr(g, i++, dst[1], 0); 8007bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams } 8017bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams iop_desc_init_pq(g, i, flags); 8027bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams } 8037bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams spin_unlock_bh(&iop_chan->lock); 8047bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 8057bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams return sw_desc ? &sw_desc->async_tx : NULL; 8067bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams} 8077bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 8087bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williamsstatic struct dma_async_tx_descriptor * 8097bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williamsiop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, 8107bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams unsigned int src_cnt, const unsigned char *scf, 8117bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams size_t len, enum sum_check_flags *pqres, 8127bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams unsigned long flags) 8137bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams{ 8147bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 8157bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams struct iop_adma_desc_slot *sw_desc, *g; 8167bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams int slot_cnt, slots_per_op; 8177bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 8187bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams if (unlikely(!len)) 8197bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams return NULL; 8207bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); 8217bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 8227bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n", 8237bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams __func__, src_cnt, len); 8247bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 8257bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams spin_lock_bh(&iop_chan->lock); 8267bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams slot_cnt = iop_chan_pq_zero_sum_slot_count(len, src_cnt + 2, &slots_per_op); 8277bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 8287bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams if (sw_desc) { 8297bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams /* for validate operations p and q are tagged onto the 8307bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams * end of the source list 8317bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams */ 8327bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams int pq_idx = src_cnt; 8337bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 8347bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams g = sw_desc->group_head; 8357bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams iop_desc_init_pq_zero_sum(g, src_cnt+2, flags); 8367bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams iop_desc_set_pq_zero_sum_byte_count(g, len); 8377bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams g->pq_check_result = pqres; 8387bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams pr_debug("\t%s: g->pq_check_result: %p\n", 8397bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams __func__, g->pq_check_result); 8407bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams sw_desc->unmap_src_cnt = src_cnt+2; 8417bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams sw_desc->unmap_len = len; 8427bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams sw_desc->async_tx.flags = flags; 8437bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams while (src_cnt--) 8447bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams iop_desc_set_pq_zero_sum_src_addr(g, src_cnt, 8457bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams src[src_cnt], 8467bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams scf[src_cnt]); 8477bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams iop_desc_set_pq_zero_sum_addr(g, pq_idx, src); 8487bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams } 8497bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams spin_unlock_bh(&iop_chan->lock); 8507bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 8517bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams return sw_desc ? &sw_desc->async_tx : NULL; 8527bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams} 8537bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams 854c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic void iop_adma_free_chan_resources(struct dma_chan *chan) 855c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 856c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 857c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_desc_slot *iter, *_iter; 858c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int in_use_descs = 0; 859c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 860c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_adma_slot_cleanup(iop_chan); 861c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 862c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_lock_bh(&iop_chan->lock); 863c211092313b90f898dec61f35207fc282d1eadc3Dan Williams list_for_each_entry_safe(iter, _iter, &iop_chan->chain, 864c211092313b90f898dec61f35207fc282d1eadc3Dan Williams chain_node) { 865c211092313b90f898dec61f35207fc282d1eadc3Dan Williams in_use_descs++; 866c211092313b90f898dec61f35207fc282d1eadc3Dan Williams list_del(&iter->chain_node); 867c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 868c211092313b90f898dec61f35207fc282d1eadc3Dan Williams list_for_each_entry_safe_reverse( 869c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iter, _iter, &iop_chan->all_slots, slot_node) { 870c211092313b90f898dec61f35207fc282d1eadc3Dan Williams list_del(&iter->slot_node); 871c211092313b90f898dec61f35207fc282d1eadc3Dan Williams kfree(iter); 872c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan->slots_allocated--; 873c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 874c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan->last_used = NULL; 875c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 876c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_dbg(iop_chan->device->common.dev, "%s slots_allocated %d\n", 8773d9b525b69bc3302d8355e5f5cf081a856c211e0Harvey Harrison __func__, iop_chan->slots_allocated); 878c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_unlock_bh(&iop_chan->lock); 879c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 880c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* one is ok since we left it on there on purpose */ 881c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (in_use_descs > 1) 882c211092313b90f898dec61f35207fc282d1eadc3Dan Williams printk(KERN_ERR "IOP: Freeing %d in use descriptors!\n", 883c211092313b90f898dec61f35207fc282d1eadc3Dan Williams in_use_descs - 1); 884c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 885c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 886c211092313b90f898dec61f35207fc282d1eadc3Dan Williams/** 8870793448187643b50af89d36b08470baf45a3cab4Linus Walleij * iop_adma_status - poll the status of an ADMA transaction 888c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * @chan: ADMA channel handle 889c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * @cookie: ADMA transaction identifier 8900793448187643b50af89d36b08470baf45a3cab4Linus Walleij * @txstate: a holder for the current state of the channel or NULL 891c211092313b90f898dec61f35207fc282d1eadc3Dan Williams */ 8920793448187643b50af89d36b08470baf45a3cab4Linus Walleijstatic enum dma_status iop_adma_status(struct dma_chan *chan, 893c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_cookie_t cookie, 8940793448187643b50af89d36b08470baf45a3cab4Linus Walleij struct dma_tx_state *txstate) 895c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 896c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 897949ff5b8d46b5e3435d21b2651ce3a2599208d44Vinod Koul int ret; 89896a2af41c78b1fbb1f567a3486bdc63f7b31c5fdRussell King - ARM Linux 89996a2af41c78b1fbb1f567a3486bdc63f7b31c5fdRussell King - ARM Linux ret = dma_cookie_status(chan, cookie, txstate); 900c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (ret == DMA_SUCCESS) 901c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return ret; 902c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 903c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_adma_slot_cleanup(iop_chan); 904c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 90596a2af41c78b1fbb1f567a3486bdc63f7b31c5fdRussell King - ARM Linux return dma_cookie_status(chan, cookie, txstate); 906c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 907c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 908c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic irqreturn_t iop_adma_eot_handler(int irq, void *data) 909c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 910c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_chan *chan = data; 911c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 9123d9b525b69bc3302d8355e5f5cf081a856c211e0Harvey Harrison dev_dbg(chan->device->common.dev, "%s\n", __func__); 913c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 914c211092313b90f898dec61f35207fc282d1eadc3Dan Williams tasklet_schedule(&chan->irq_tasklet); 915c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 916c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_adma_device_clear_eot_status(chan); 917c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 918c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return IRQ_HANDLED; 919c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 920c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 921c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic irqreturn_t iop_adma_eoc_handler(int irq, void *data) 922c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 923c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_chan *chan = data; 924c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 9253d9b525b69bc3302d8355e5f5cf081a856c211e0Harvey Harrison dev_dbg(chan->device->common.dev, "%s\n", __func__); 926c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 927c211092313b90f898dec61f35207fc282d1eadc3Dan Williams tasklet_schedule(&chan->irq_tasklet); 928c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 929c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_adma_device_clear_eoc_status(chan); 930c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 931c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return IRQ_HANDLED; 932c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 933c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 934c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic irqreturn_t iop_adma_err_handler(int irq, void *data) 935c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 936c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_chan *chan = data; 937c211092313b90f898dec61f35207fc282d1eadc3Dan Williams unsigned long status = iop_chan_get_status(chan); 938c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 939c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_printk(KERN_ERR, chan->device->common.dev, 940c211092313b90f898dec61f35207fc282d1eadc3Dan Williams "error ( %s%s%s%s%s%s%s)\n", 941c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_is_err_int_parity(status, chan) ? "int_parity " : "", 942c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_is_err_mcu_abort(status, chan) ? "mcu_abort " : "", 943c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_is_err_int_tabort(status, chan) ? "int_tabort " : "", 944c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_is_err_int_mabort(status, chan) ? "int_mabort " : "", 945c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_is_err_pci_tabort(status, chan) ? "pci_tabort " : "", 946c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_is_err_pci_mabort(status, chan) ? "pci_mabort " : "", 947c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_is_err_split_tx(status, chan) ? "split_tx " : ""); 948c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 949c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_adma_device_clear_err_status(chan); 950c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 951c211092313b90f898dec61f35207fc282d1eadc3Dan Williams BUG(); 952c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 953c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return IRQ_HANDLED; 954c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 955c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 956c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic void iop_adma_issue_pending(struct dma_chan *chan) 957c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 958c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 959c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 960c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (iop_chan->pending) { 961c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan->pending = 0; 962c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan_append(iop_chan); 963c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 964c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 965c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 966c211092313b90f898dec61f35207fc282d1eadc3Dan Williams/* 967c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * Perform a transaction to verify the HW works. 968c211092313b90f898dec61f35207fc282d1eadc3Dan Williams */ 969c211092313b90f898dec61f35207fc282d1eadc3Dan Williams#define IOP_ADMA_TEST_SIZE 2000 970c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 971c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic int __devinit iop_adma_memcpy_self_test(struct iop_adma_device *device) 972c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 973c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int i; 974c211092313b90f898dec61f35207fc282d1eadc3Dan Williams void *src, *dest; 975c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_addr_t src_dma, dest_dma; 976c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct dma_chan *dma_chan; 977c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_cookie_t cookie; 978c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct dma_async_tx_descriptor *tx; 979c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int err = 0; 980c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_chan *iop_chan; 981c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 9823d9b525b69bc3302d8355e5f5cf081a856c211e0Harvey Harrison dev_dbg(device->common.dev, "%s\n", __func__); 983c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 984eccf2144e1232c33a8235033ffa079b6ebf92fafChristophe Jaillet src = kmalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL); 985c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (!src) 986c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return -ENOMEM; 987eccf2144e1232c33a8235033ffa079b6ebf92fafChristophe Jaillet dest = kzalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL); 988c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (!dest) { 989c211092313b90f898dec61f35207fc282d1eadc3Dan Williams kfree(src); 990c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return -ENOMEM; 991c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 992c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 993c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* Fill in src buffer */ 994c211092313b90f898dec61f35207fc282d1eadc3Dan Williams for (i = 0; i < IOP_ADMA_TEST_SIZE; i++) 995c211092313b90f898dec61f35207fc282d1eadc3Dan Williams ((u8 *) src)[i] = (u8)i; 996c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 997c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* Start copy, using first DMA channel */ 998c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_chan = container_of(device->common.channels.next, 999c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct dma_chan, 1000c211092313b90f898dec61f35207fc282d1eadc3Dan Williams device_node); 1001aa1e6f1a385eb2b04171ec841f3b760091e4a8eeDan Williams if (iop_adma_alloc_chan_resources(dma_chan) < 1) { 1002c211092313b90f898dec61f35207fc282d1eadc3Dan Williams err = -ENODEV; 1003c211092313b90f898dec61f35207fc282d1eadc3Dan Williams goto out; 1004c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1005c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1006c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dest_dma = dma_map_single(dma_chan->device->dev, dest, 1007c211092313b90f898dec61f35207fc282d1eadc3Dan Williams IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE); 1008c211092313b90f898dec61f35207fc282d1eadc3Dan Williams src_dma = dma_map_single(dma_chan->device->dev, src, 1009c211092313b90f898dec61f35207fc282d1eadc3Dan Williams IOP_ADMA_TEST_SIZE, DMA_TO_DEVICE); 10100036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams tx = iop_adma_prep_dma_memcpy(dma_chan, dest_dma, src_dma, 1011636bdeaa1243327501edfd2a597ed7443eb4239aDan Williams IOP_ADMA_TEST_SIZE, 1012636bdeaa1243327501edfd2a597ed7443eb4239aDan Williams DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1013c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1014c211092313b90f898dec61f35207fc282d1eadc3Dan Williams cookie = iop_adma_tx_submit(tx); 1015c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_adma_issue_pending(dma_chan); 1016c211092313b90f898dec61f35207fc282d1eadc3Dan Williams msleep(1); 1017c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 10180793448187643b50af89d36b08470baf45a3cab4Linus Walleij if (iop_adma_status(dma_chan, cookie, NULL) != 1019c211092313b90f898dec61f35207fc282d1eadc3Dan Williams DMA_SUCCESS) { 1020c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_printk(KERN_ERR, dma_chan->device->dev, 1021c211092313b90f898dec61f35207fc282d1eadc3Dan Williams "Self-test copy timed out, disabling\n"); 1022c211092313b90f898dec61f35207fc282d1eadc3Dan Williams err = -ENODEV; 1023c211092313b90f898dec61f35207fc282d1eadc3Dan Williams goto free_resources; 1024c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1025c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1026c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan = to_iop_adma_chan(dma_chan); 1027c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma, 1028c211092313b90f898dec61f35207fc282d1eadc3Dan Williams IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE); 1029c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (memcmp(src, dest, IOP_ADMA_TEST_SIZE)) { 1030c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_printk(KERN_ERR, dma_chan->device->dev, 1031c211092313b90f898dec61f35207fc282d1eadc3Dan Williams "Self-test copy failed compare, disabling\n"); 1032c211092313b90f898dec61f35207fc282d1eadc3Dan Williams err = -ENODEV; 1033c211092313b90f898dec61f35207fc282d1eadc3Dan Williams goto free_resources; 1034c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1035c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1036c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsfree_resources: 1037c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_adma_free_chan_resources(dma_chan); 1038c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsout: 1039c211092313b90f898dec61f35207fc282d1eadc3Dan Williams kfree(src); 1040c211092313b90f898dec61f35207fc282d1eadc3Dan Williams kfree(dest); 1041c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return err; 1042c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 1043c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1044c211092313b90f898dec61f35207fc282d1eadc3Dan Williams#define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */ 1045c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic int __devinit 1046099f53cb50e45ef617a9f1d63ceec799e489418bDan Williamsiop_adma_xor_val_self_test(struct iop_adma_device *device) 1047c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 1048c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int i, src_idx; 1049c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct page *dest; 1050c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct page *xor_srcs[IOP_ADMA_NUM_SRC_TEST]; 1051c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct page *zero_sum_srcs[IOP_ADMA_NUM_SRC_TEST + 1]; 10520036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_addr_t dma_srcs[IOP_ADMA_NUM_SRC_TEST + 1]; 1053c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_addr_t dma_addr, dest_dma; 1054c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct dma_async_tx_descriptor *tx; 1055c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct dma_chan *dma_chan; 1056c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_cookie_t cookie; 1057c211092313b90f898dec61f35207fc282d1eadc3Dan Williams u8 cmp_byte = 0; 1058c211092313b90f898dec61f35207fc282d1eadc3Dan Williams u32 cmp_word; 1059c211092313b90f898dec61f35207fc282d1eadc3Dan Williams u32 zero_sum_result; 1060c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int err = 0; 1061c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_chan *iop_chan; 1062c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 10633d9b525b69bc3302d8355e5f5cf081a856c211e0Harvey Harrison dev_dbg(device->common.dev, "%s\n", __func__); 1064c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1065c211092313b90f898dec61f35207fc282d1eadc3Dan Williams for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) { 1066c211092313b90f898dec61f35207fc282d1eadc3Dan Williams xor_srcs[src_idx] = alloc_page(GFP_KERNEL); 1067a09b09ae51ace43d28cd9bc1c8bb97986f2b55a6Roel Kluin if (!xor_srcs[src_idx]) { 1068a09b09ae51ace43d28cd9bc1c8bb97986f2b55a6Roel Kluin while (src_idx--) 1069c211092313b90f898dec61f35207fc282d1eadc3Dan Williams __free_page(xor_srcs[src_idx]); 1070a09b09ae51ace43d28cd9bc1c8bb97986f2b55a6Roel Kluin return -ENOMEM; 1071a09b09ae51ace43d28cd9bc1c8bb97986f2b55a6Roel Kluin } 1072c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1073c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1074c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dest = alloc_page(GFP_KERNEL); 1075a09b09ae51ace43d28cd9bc1c8bb97986f2b55a6Roel Kluin if (!dest) { 1076a09b09ae51ace43d28cd9bc1c8bb97986f2b55a6Roel Kluin while (src_idx--) 1077c211092313b90f898dec61f35207fc282d1eadc3Dan Williams __free_page(xor_srcs[src_idx]); 1078a09b09ae51ace43d28cd9bc1c8bb97986f2b55a6Roel Kluin return -ENOMEM; 1079a09b09ae51ace43d28cd9bc1c8bb97986f2b55a6Roel Kluin } 1080c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1081c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* Fill in src buffers */ 1082c211092313b90f898dec61f35207fc282d1eadc3Dan Williams for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) { 1083c211092313b90f898dec61f35207fc282d1eadc3Dan Williams u8 *ptr = page_address(xor_srcs[src_idx]); 1084c211092313b90f898dec61f35207fc282d1eadc3Dan Williams for (i = 0; i < PAGE_SIZE; i++) 1085c211092313b90f898dec61f35207fc282d1eadc3Dan Williams ptr[i] = (1 << src_idx); 1086c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1087c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1088c211092313b90f898dec61f35207fc282d1eadc3Dan Williams for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) 1089c211092313b90f898dec61f35207fc282d1eadc3Dan Williams cmp_byte ^= (u8) (1 << src_idx); 1090c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1091c211092313b90f898dec61f35207fc282d1eadc3Dan Williams cmp_word = (cmp_byte << 24) | (cmp_byte << 16) | 1092c211092313b90f898dec61f35207fc282d1eadc3Dan Williams (cmp_byte << 8) | cmp_byte; 1093c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1094c211092313b90f898dec61f35207fc282d1eadc3Dan Williams memset(page_address(dest), 0, PAGE_SIZE); 1095c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1096c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_chan = container_of(device->common.channels.next, 1097c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct dma_chan, 1098c211092313b90f898dec61f35207fc282d1eadc3Dan Williams device_node); 1099aa1e6f1a385eb2b04171ec841f3b760091e4a8eeDan Williams if (iop_adma_alloc_chan_resources(dma_chan) < 1) { 1100c211092313b90f898dec61f35207fc282d1eadc3Dan Williams err = -ENODEV; 1101c211092313b90f898dec61f35207fc282d1eadc3Dan Williams goto out; 1102c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1103c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1104c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* test xor */ 1105c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dest_dma = dma_map_page(dma_chan->device->dev, dest, 0, 1106c211092313b90f898dec61f35207fc282d1eadc3Dan Williams PAGE_SIZE, DMA_FROM_DEVICE); 11070036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) 11080036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_srcs[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i], 11090036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams 0, PAGE_SIZE, DMA_TO_DEVICE); 11100036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams tx = iop_adma_prep_dma_xor(dma_chan, dest_dma, dma_srcs, 1111636bdeaa1243327501edfd2a597ed7443eb4239aDan Williams IOP_ADMA_NUM_SRC_TEST, PAGE_SIZE, 1112636bdeaa1243327501edfd2a597ed7443eb4239aDan Williams DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1113c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1114c211092313b90f898dec61f35207fc282d1eadc3Dan Williams cookie = iop_adma_tx_submit(tx); 1115c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_adma_issue_pending(dma_chan); 1116c211092313b90f898dec61f35207fc282d1eadc3Dan Williams msleep(8); 1117c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 11180793448187643b50af89d36b08470baf45a3cab4Linus Walleij if (iop_adma_status(dma_chan, cookie, NULL) != 1119c211092313b90f898dec61f35207fc282d1eadc3Dan Williams DMA_SUCCESS) { 1120c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_printk(KERN_ERR, dma_chan->device->dev, 1121c211092313b90f898dec61f35207fc282d1eadc3Dan Williams "Self-test xor timed out, disabling\n"); 1122c211092313b90f898dec61f35207fc282d1eadc3Dan Williams err = -ENODEV; 1123c211092313b90f898dec61f35207fc282d1eadc3Dan Williams goto free_resources; 1124c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1125c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1126c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan = to_iop_adma_chan(dma_chan); 1127c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma, 1128c211092313b90f898dec61f35207fc282d1eadc3Dan Williams PAGE_SIZE, DMA_FROM_DEVICE); 1129c211092313b90f898dec61f35207fc282d1eadc3Dan Williams for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) { 1130c211092313b90f898dec61f35207fc282d1eadc3Dan Williams u32 *ptr = page_address(dest); 1131c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (ptr[i] != cmp_word) { 1132c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_printk(KERN_ERR, dma_chan->device->dev, 1133c211092313b90f898dec61f35207fc282d1eadc3Dan Williams "Self-test xor failed compare, disabling\n"); 1134c211092313b90f898dec61f35207fc282d1eadc3Dan Williams err = -ENODEV; 1135c211092313b90f898dec61f35207fc282d1eadc3Dan Williams goto free_resources; 1136c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1137c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1138c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_sync_single_for_device(&iop_chan->device->pdev->dev, dest_dma, 1139c211092313b90f898dec61f35207fc282d1eadc3Dan Williams PAGE_SIZE, DMA_TO_DEVICE); 1140c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1141c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* skip zero sum if the capability is not present */ 1142099f53cb50e45ef617a9f1d63ceec799e489418bDan Williams if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask)) 1143c211092313b90f898dec61f35207fc282d1eadc3Dan Williams goto free_resources; 1144c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1145c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* zero sum the sources with the destintation page */ 1146c211092313b90f898dec61f35207fc282d1eadc3Dan Williams for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) 1147c211092313b90f898dec61f35207fc282d1eadc3Dan Williams zero_sum_srcs[i] = xor_srcs[i]; 1148c211092313b90f898dec61f35207fc282d1eadc3Dan Williams zero_sum_srcs[i] = dest; 1149c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1150c211092313b90f898dec61f35207fc282d1eadc3Dan Williams zero_sum_result = 1; 1151c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 11520036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++) 11530036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_srcs[i] = dma_map_page(dma_chan->device->dev, 11540036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams zero_sum_srcs[i], 0, PAGE_SIZE, 11550036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams DMA_TO_DEVICE); 1156099f53cb50e45ef617a9f1d63ceec799e489418bDan Williams tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs, 1157099f53cb50e45ef617a9f1d63ceec799e489418bDan Williams IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE, 1158099f53cb50e45ef617a9f1d63ceec799e489418bDan Williams &zero_sum_result, 1159099f53cb50e45ef617a9f1d63ceec799e489418bDan Williams DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1160c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1161c211092313b90f898dec61f35207fc282d1eadc3Dan Williams cookie = iop_adma_tx_submit(tx); 1162c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_adma_issue_pending(dma_chan); 1163c211092313b90f898dec61f35207fc282d1eadc3Dan Williams msleep(8); 1164c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 11650793448187643b50af89d36b08470baf45a3cab4Linus Walleij if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { 1166c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_printk(KERN_ERR, dma_chan->device->dev, 1167c211092313b90f898dec61f35207fc282d1eadc3Dan Williams "Self-test zero sum timed out, disabling\n"); 1168c211092313b90f898dec61f35207fc282d1eadc3Dan Williams err = -ENODEV; 1169c211092313b90f898dec61f35207fc282d1eadc3Dan Williams goto free_resources; 1170c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1171c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1172c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (zero_sum_result != 0) { 1173c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_printk(KERN_ERR, dma_chan->device->dev, 1174c211092313b90f898dec61f35207fc282d1eadc3Dan Williams "Self-test zero sum failed compare, disabling\n"); 1175c211092313b90f898dec61f35207fc282d1eadc3Dan Williams err = -ENODEV; 1176c211092313b90f898dec61f35207fc282d1eadc3Dan Williams goto free_resources; 1177c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1178c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1179c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* test memset */ 1180c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_addr = dma_map_page(dma_chan->device->dev, dest, 0, 1181c211092313b90f898dec61f35207fc282d1eadc3Dan Williams PAGE_SIZE, DMA_FROM_DEVICE); 1182636bdeaa1243327501edfd2a597ed7443eb4239aDan Williams tx = iop_adma_prep_dma_memset(dma_chan, dma_addr, 0, PAGE_SIZE, 1183636bdeaa1243327501edfd2a597ed7443eb4239aDan Williams DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1184c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1185c211092313b90f898dec61f35207fc282d1eadc3Dan Williams cookie = iop_adma_tx_submit(tx); 1186c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_adma_issue_pending(dma_chan); 1187c211092313b90f898dec61f35207fc282d1eadc3Dan Williams msleep(8); 1188c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 11890793448187643b50af89d36b08470baf45a3cab4Linus Walleij if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { 1190c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_printk(KERN_ERR, dma_chan->device->dev, 1191c211092313b90f898dec61f35207fc282d1eadc3Dan Williams "Self-test memset timed out, disabling\n"); 1192c211092313b90f898dec61f35207fc282d1eadc3Dan Williams err = -ENODEV; 1193c211092313b90f898dec61f35207fc282d1eadc3Dan Williams goto free_resources; 1194c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1195c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1196c211092313b90f898dec61f35207fc282d1eadc3Dan Williams for (i = 0; i < PAGE_SIZE/sizeof(u32); i++) { 1197c211092313b90f898dec61f35207fc282d1eadc3Dan Williams u32 *ptr = page_address(dest); 1198c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (ptr[i]) { 1199c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_printk(KERN_ERR, dma_chan->device->dev, 1200c211092313b90f898dec61f35207fc282d1eadc3Dan Williams "Self-test memset failed compare, disabling\n"); 1201c211092313b90f898dec61f35207fc282d1eadc3Dan Williams err = -ENODEV; 1202c211092313b90f898dec61f35207fc282d1eadc3Dan Williams goto free_resources; 1203c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1204c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1205c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1206c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* test for non-zero parity sum */ 1207c211092313b90f898dec61f35207fc282d1eadc3Dan Williams zero_sum_result = 0; 12080036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++) 12090036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams dma_srcs[i] = dma_map_page(dma_chan->device->dev, 12100036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams zero_sum_srcs[i], 0, PAGE_SIZE, 12110036731c88fdb5bf4f04a796a30b5e445fc57f54Dan Williams DMA_TO_DEVICE); 1212099f53cb50e45ef617a9f1d63ceec799e489418bDan Williams tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs, 1213099f53cb50e45ef617a9f1d63ceec799e489418bDan Williams IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE, 1214099f53cb50e45ef617a9f1d63ceec799e489418bDan Williams &zero_sum_result, 1215099f53cb50e45ef617a9f1d63ceec799e489418bDan Williams DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1216c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1217c211092313b90f898dec61f35207fc282d1eadc3Dan Williams cookie = iop_adma_tx_submit(tx); 1218c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_adma_issue_pending(dma_chan); 1219c211092313b90f898dec61f35207fc282d1eadc3Dan Williams msleep(8); 1220c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 12210793448187643b50af89d36b08470baf45a3cab4Linus Walleij if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { 1222c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_printk(KERN_ERR, dma_chan->device->dev, 1223c211092313b90f898dec61f35207fc282d1eadc3Dan Williams "Self-test non-zero sum timed out, disabling\n"); 1224c211092313b90f898dec61f35207fc282d1eadc3Dan Williams err = -ENODEV; 1225c211092313b90f898dec61f35207fc282d1eadc3Dan Williams goto free_resources; 1226c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1227c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1228c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (zero_sum_result != 1) { 1229c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_printk(KERN_ERR, dma_chan->device->dev, 1230c211092313b90f898dec61f35207fc282d1eadc3Dan Williams "Self-test non-zero sum failed compare, disabling\n"); 1231c211092313b90f898dec61f35207fc282d1eadc3Dan Williams err = -ENODEV; 1232c211092313b90f898dec61f35207fc282d1eadc3Dan Williams goto free_resources; 1233c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1234c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1235c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsfree_resources: 1236c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_adma_free_chan_resources(dma_chan); 1237c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsout: 1238c211092313b90f898dec61f35207fc282d1eadc3Dan Williams src_idx = IOP_ADMA_NUM_SRC_TEST; 1239c211092313b90f898dec61f35207fc282d1eadc3Dan Williams while (src_idx--) 1240c211092313b90f898dec61f35207fc282d1eadc3Dan Williams __free_page(xor_srcs[src_idx]); 1241c211092313b90f898dec61f35207fc282d1eadc3Dan Williams __free_page(dest); 1242c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return err; 1243c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 1244c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 12450261f7416362f6affc2d4fe7fea9320a6bdaaee6Wei Yongquan#ifdef CONFIG_RAID6_PQ 1246f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williamsstatic int __devinit 1247f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williamsiop_adma_pq_zero_sum_self_test(struct iop_adma_device *device) 1248f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams{ 1249f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams /* combined sources, software pq results, and extra hw pq results */ 1250f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams struct page *pq[IOP_ADMA_NUM_SRC_TEST+2+2]; 1251f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams /* ptr to the extra hw pq buffers defined above */ 1252f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams struct page **pq_hw = &pq[IOP_ADMA_NUM_SRC_TEST+2]; 1253f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams /* address conversion buffers (dma_map / page_address) */ 1254f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams void *pq_sw[IOP_ADMA_NUM_SRC_TEST+2]; 12553d9ea9e3af048ab6b8dced15248384e548ba05eaDon Morris dma_addr_t pq_src[IOP_ADMA_NUM_SRC_TEST+2]; 12563d9ea9e3af048ab6b8dced15248384e548ba05eaDon Morris dma_addr_t *pq_dest = &pq_src[IOP_ADMA_NUM_SRC_TEST]; 1257f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1258f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams int i; 1259f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams struct dma_async_tx_descriptor *tx; 1260f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams struct dma_chan *dma_chan; 1261f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams dma_cookie_t cookie; 1262f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams u32 zero_sum_result; 1263f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams int err = 0; 1264f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams struct device *dev; 1265f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1266f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams dev_dbg(device->common.dev, "%s\n", __func__); 1267f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1268f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams for (i = 0; i < ARRAY_SIZE(pq); i++) { 1269f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams pq[i] = alloc_page(GFP_KERNEL); 1270f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams if (!pq[i]) { 1271f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams while (i--) 1272f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams __free_page(pq[i]); 1273f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams return -ENOMEM; 1274f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams } 1275f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams } 1276f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1277f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams /* Fill in src buffers */ 1278f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) { 1279f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams pq_sw[i] = page_address(pq[i]); 1280f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams memset(pq_sw[i], 0x11111111 * (1<<i), PAGE_SIZE); 1281f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams } 1282f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams pq_sw[i] = page_address(pq[i]); 1283f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams pq_sw[i+1] = page_address(pq[i+1]); 1284f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1285f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams dma_chan = container_of(device->common.channels.next, 1286f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams struct dma_chan, 1287f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams device_node); 1288f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams if (iop_adma_alloc_chan_resources(dma_chan) < 1) { 1289f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams err = -ENODEV; 1290f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams goto out; 1291f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams } 1292f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1293f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams dev = dma_chan->device->dev; 1294f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1295f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams /* initialize the dests */ 1296f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams memset(page_address(pq_hw[0]), 0 , PAGE_SIZE); 1297f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams memset(page_address(pq_hw[1]), 0 , PAGE_SIZE); 1298f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1299f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams /* test pq */ 1300f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams pq_dest[0] = dma_map_page(dev, pq_hw[0], 0, PAGE_SIZE, DMA_FROM_DEVICE); 1301f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams pq_dest[1] = dma_map_page(dev, pq_hw[1], 0, PAGE_SIZE, DMA_FROM_DEVICE); 1302f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) 1303f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, 1304f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams DMA_TO_DEVICE); 1305f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1306f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams tx = iop_adma_prep_dma_pq(dma_chan, pq_dest, pq_src, 1307f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams IOP_ADMA_NUM_SRC_TEST, (u8 *)raid6_gfexp, 1308f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams PAGE_SIZE, 1309f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams DMA_PREP_INTERRUPT | 1310f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams DMA_CTRL_ACK); 1311f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1312f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams cookie = iop_adma_tx_submit(tx); 1313f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams iop_adma_issue_pending(dma_chan); 1314f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams msleep(8); 1315f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 13160793448187643b50af89d36b08470baf45a3cab4Linus Walleij if (iop_adma_status(dma_chan, cookie, NULL) != 1317f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams DMA_SUCCESS) { 1318f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams dev_err(dev, "Self-test pq timed out, disabling\n"); 1319f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams err = -ENODEV; 1320f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams goto free_resources; 1321f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams } 1322f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1323f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams raid6_call.gen_syndrome(IOP_ADMA_NUM_SRC_TEST+2, PAGE_SIZE, pq_sw); 1324f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1325f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST], 1326f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams page_address(pq_hw[0]), PAGE_SIZE) != 0) { 1327f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams dev_err(dev, "Self-test p failed compare, disabling\n"); 1328f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams err = -ENODEV; 1329f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams goto free_resources; 1330f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams } 1331f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST+1], 1332f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams page_address(pq_hw[1]), PAGE_SIZE) != 0) { 1333f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams dev_err(dev, "Self-test q failed compare, disabling\n"); 1334f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams err = -ENODEV; 1335f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams goto free_resources; 1336f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams } 1337f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1338f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams /* test correct zero sum using the software generated pq values */ 1339f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++) 1340f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, 1341f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams DMA_TO_DEVICE); 1342f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1343f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams zero_sum_result = ~0; 1344f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST], 1345f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams pq_src, IOP_ADMA_NUM_SRC_TEST, 1346f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams raid6_gfexp, PAGE_SIZE, &zero_sum_result, 1347f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams DMA_PREP_INTERRUPT|DMA_CTRL_ACK); 1348f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1349f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams cookie = iop_adma_tx_submit(tx); 1350f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams iop_adma_issue_pending(dma_chan); 1351f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams msleep(8); 1352f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 13530793448187643b50af89d36b08470baf45a3cab4Linus Walleij if (iop_adma_status(dma_chan, cookie, NULL) != 1354f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams DMA_SUCCESS) { 1355f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n"); 1356f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams err = -ENODEV; 1357f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams goto free_resources; 1358f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams } 1359f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1360f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams if (zero_sum_result != 0) { 1361f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams dev_err(dev, "Self-test pq-zero-sum failed to validate: %x\n", 1362f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams zero_sum_result); 1363f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams err = -ENODEV; 1364f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams goto free_resources; 1365f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams } 1366f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1367f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams /* test incorrect zero sum */ 1368f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams i = IOP_ADMA_NUM_SRC_TEST; 1369f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams memset(pq_sw[i] + 100, 0, 100); 1370f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams memset(pq_sw[i+1] + 200, 0, 200); 1371f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++) 1372f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, 1373f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams DMA_TO_DEVICE); 1374f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1375f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams zero_sum_result = 0; 1376f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST], 1377f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams pq_src, IOP_ADMA_NUM_SRC_TEST, 1378f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams raid6_gfexp, PAGE_SIZE, &zero_sum_result, 1379f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams DMA_PREP_INTERRUPT|DMA_CTRL_ACK); 1380f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1381f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams cookie = iop_adma_tx_submit(tx); 1382f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams iop_adma_issue_pending(dma_chan); 1383f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams msleep(8); 1384f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 13850793448187643b50af89d36b08470baf45a3cab4Linus Walleij if (iop_adma_status(dma_chan, cookie, NULL) != 1386f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams DMA_SUCCESS) { 1387f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n"); 1388f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams err = -ENODEV; 1389f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams goto free_resources; 1390f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams } 1391f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1392f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams if (zero_sum_result != (SUM_CHECK_P_RESULT | SUM_CHECK_Q_RESULT)) { 1393f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams dev_err(dev, "Self-test !pq-zero-sum failed to validate: %x\n", 1394f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams zero_sum_result); 1395f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams err = -ENODEV; 1396f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams goto free_resources; 1397f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams } 1398f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1399f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williamsfree_resources: 1400f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams iop_adma_free_chan_resources(dma_chan); 1401f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williamsout: 1402f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams i = ARRAY_SIZE(pq); 1403f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams while (i--) 1404f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams __free_page(pq[i]); 1405f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams return err; 1406f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams} 1407f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams#endif 1408f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1409c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic int __devexit iop_adma_remove(struct platform_device *dev) 1410c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 1411c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_device *device = platform_get_drvdata(dev); 1412c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct dma_chan *chan, *_chan; 1413c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_chan *iop_chan; 1414c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_platform_data *plat_data = dev->dev.platform_data; 1415c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1416c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_async_device_unregister(&device->common); 1417c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1418c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_free_coherent(&dev->dev, plat_data->pool_size, 1419c211092313b90f898dec61f35207fc282d1eadc3Dan Williams device->dma_desc_pool_virt, device->dma_desc_pool); 1420c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1421c211092313b90f898dec61f35207fc282d1eadc3Dan Williams list_for_each_entry_safe(chan, _chan, &device->common.channels, 1422c211092313b90f898dec61f35207fc282d1eadc3Dan Williams device_node) { 1423c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan = to_iop_adma_chan(chan); 1424c211092313b90f898dec61f35207fc282d1eadc3Dan Williams list_del(&chan->device_node); 1425c211092313b90f898dec61f35207fc282d1eadc3Dan Williams kfree(iop_chan); 1426c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1427c211092313b90f898dec61f35207fc282d1eadc3Dan Williams kfree(device); 1428c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1429c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return 0; 1430c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 1431c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1432c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic int __devinit iop_adma_probe(struct platform_device *pdev) 1433c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 1434c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct resource *res; 1435c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int ret = 0, i; 1436c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_device *adev; 1437c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_chan *iop_chan; 1438c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct dma_device *dma_dev; 1439c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_platform_data *plat_data = pdev->dev.platform_data; 1440c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1441c211092313b90f898dec61f35207fc282d1eadc3Dan Williams res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1442c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (!res) 1443c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return -ENODEV; 1444c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1445c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (!devm_request_mem_region(&pdev->dev, res->start, 14462e032b62c4c8560d6416ad3cc925cfc2a5eafb07H Hartley Sweeten resource_size(res), pdev->name)) 1447c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return -EBUSY; 1448c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1449c211092313b90f898dec61f35207fc282d1eadc3Dan Williams adev = kzalloc(sizeof(*adev), GFP_KERNEL); 1450c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (!adev) 1451c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return -ENOMEM; 1452c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_dev = &adev->common; 1453c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1454c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* allocate coherent memory for hardware descriptors 1455c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * note: writecombine gives slightly better performance, but 1456c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * requires that we explicitly flush the writes 1457c211092313b90f898dec61f35207fc282d1eadc3Dan Williams */ 1458c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if ((adev->dma_desc_pool_virt = dma_alloc_writecombine(&pdev->dev, 1459c211092313b90f898dec61f35207fc282d1eadc3Dan Williams plat_data->pool_size, 1460c211092313b90f898dec61f35207fc282d1eadc3Dan Williams &adev->dma_desc_pool, 1461c211092313b90f898dec61f35207fc282d1eadc3Dan Williams GFP_KERNEL)) == NULL) { 1462c211092313b90f898dec61f35207fc282d1eadc3Dan Williams ret = -ENOMEM; 1463c211092313b90f898dec61f35207fc282d1eadc3Dan Williams goto err_free_adev; 1464c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1465c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1466b6695e411f46036bcaab29908e8aa46fbbe101edMasanari Iida dev_dbg(&pdev->dev, "%s: allocated descriptor pool virt %p phys %p\n", 14673d9b525b69bc3302d8355e5f5cf081a856c211e0Harvey Harrison __func__, adev->dma_desc_pool_virt, 1468c211092313b90f898dec61f35207fc282d1eadc3Dan Williams (void *) adev->dma_desc_pool); 1469c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1470c211092313b90f898dec61f35207fc282d1eadc3Dan Williams adev->id = plat_data->hw_id; 1471c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1472c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* discover transaction capabilites from the platform data */ 1473c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_dev->cap_mask = plat_data->cap_mask; 1474c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1475c211092313b90f898dec61f35207fc282d1eadc3Dan Williams adev->pdev = pdev; 1476c211092313b90f898dec61f35207fc282d1eadc3Dan Williams platform_set_drvdata(pdev, adev); 1477c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1478c211092313b90f898dec61f35207fc282d1eadc3Dan Williams INIT_LIST_HEAD(&dma_dev->channels); 1479c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1480c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* set base routines */ 1481c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_dev->device_alloc_chan_resources = iop_adma_alloc_chan_resources; 1482c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_dev->device_free_chan_resources = iop_adma_free_chan_resources; 14830793448187643b50af89d36b08470baf45a3cab4Linus Walleij dma_dev->device_tx_status = iop_adma_status; 1484c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_dev->device_issue_pending = iop_adma_issue_pending; 1485c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_dev->dev = &pdev->dev; 1486c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1487c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* set prep routines based on capability */ 1488c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) 1489c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_dev->device_prep_dma_memcpy = iop_adma_prep_dma_memcpy; 1490c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) 1491c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_dev->device_prep_dma_memset = iop_adma_prep_dma_memset; 1492c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 1493c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_dev->max_xor = iop_adma_get_max_xor(); 1494c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor; 1495c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1496099f53cb50e45ef617a9f1d63ceec799e489418bDan Williams if (dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask)) 1497099f53cb50e45ef617a9f1d63ceec799e489418bDan Williams dma_dev->device_prep_dma_xor_val = 1498099f53cb50e45ef617a9f1d63ceec799e489418bDan Williams iop_adma_prep_dma_xor_val; 14997bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { 15007bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams dma_set_maxpq(dma_dev, iop_adma_get_max_pq(), 0); 15017bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams dma_dev->device_prep_dma_pq = iop_adma_prep_dma_pq; 15027bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams } 15037bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams if (dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) 15047bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams dma_dev->device_prep_dma_pq_val = 15057bf649aee8ac93ecc280f8745dcf8ec19d7b9fb1Dan Williams iop_adma_prep_dma_pq_val; 1506c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask)) 1507c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_dev->device_prep_dma_interrupt = 1508c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_adma_prep_dma_interrupt; 1509c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1510c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan = kzalloc(sizeof(*iop_chan), GFP_KERNEL); 1511c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (!iop_chan) { 1512c211092313b90f898dec61f35207fc282d1eadc3Dan Williams ret = -ENOMEM; 1513c211092313b90f898dec61f35207fc282d1eadc3Dan Williams goto err_free_dma; 1514c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1515c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan->device = adev; 1516c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1517c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start, 15182e032b62c4c8560d6416ad3cc925cfc2a5eafb07H Hartley Sweeten resource_size(res)); 1519c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (!iop_chan->mmr_base) { 1520c211092313b90f898dec61f35207fc282d1eadc3Dan Williams ret = -ENOMEM; 1521c211092313b90f898dec61f35207fc282d1eadc3Dan Williams goto err_free_iop_chan; 1522c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1523c211092313b90f898dec61f35207fc282d1eadc3Dan Williams tasklet_init(&iop_chan->irq_tasklet, iop_adma_tasklet, (unsigned long) 1524c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan); 1525c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1526c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* clear errors before enabling interrupts */ 1527c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_adma_device_clear_err_status(iop_chan); 1528c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1529c211092313b90f898dec61f35207fc282d1eadc3Dan Williams for (i = 0; i < 3; i++) { 1530c211092313b90f898dec61f35207fc282d1eadc3Dan Williams irq_handler_t handler[] = { iop_adma_eot_handler, 1531c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_adma_eoc_handler, 1532c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_adma_err_handler }; 1533c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int irq = platform_get_irq(pdev, i); 1534c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (irq < 0) { 1535c211092313b90f898dec61f35207fc282d1eadc3Dan Williams ret = -ENXIO; 1536c211092313b90f898dec61f35207fc282d1eadc3Dan Williams goto err_free_iop_chan; 1537c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } else { 1538c211092313b90f898dec61f35207fc282d1eadc3Dan Williams ret = devm_request_irq(&pdev->dev, irq, 1539c211092313b90f898dec61f35207fc282d1eadc3Dan Williams handler[i], 0, pdev->name, iop_chan); 1540c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (ret) 1541c211092313b90f898dec61f35207fc282d1eadc3Dan Williams goto err_free_iop_chan; 1542c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1543c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1544c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1545c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_lock_init(&iop_chan->lock); 1546c211092313b90f898dec61f35207fc282d1eadc3Dan Williams INIT_LIST_HEAD(&iop_chan->chain); 1547c211092313b90f898dec61f35207fc282d1eadc3Dan Williams INIT_LIST_HEAD(&iop_chan->all_slots); 1548c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan->common.device = dma_dev; 15498ac695463f37af902e953d575d3f782e32e170daRussell King - ARM Linux dma_cookie_init(&iop_chan->common); 1550c211092313b90f898dec61f35207fc282d1eadc3Dan Williams list_add_tail(&iop_chan->common.device_node, &dma_dev->channels); 1551c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1552c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { 1553c211092313b90f898dec61f35207fc282d1eadc3Dan Williams ret = iop_adma_memcpy_self_test(adev); 1554c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret); 1555c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (ret) 1556c211092313b90f898dec61f35207fc282d1eadc3Dan Williams goto err_free_iop_chan; 1557c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1558c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1559c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (dma_has_cap(DMA_XOR, dma_dev->cap_mask) || 1560f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) { 1561099f53cb50e45ef617a9f1d63ceec799e489418bDan Williams ret = iop_adma_xor_val_self_test(adev); 1562c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_dbg(&pdev->dev, "xor self test returned %d\n", ret); 1563c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (ret) 1564c211092313b90f898dec61f35207fc282d1eadc3Dan Williams goto err_free_iop_chan; 1565c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } 1566c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1567f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) && 1568f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) { 15690261f7416362f6affc2d4fe7fea9320a6bdaaee6Wei Yongquan #ifdef CONFIG_RAID6_PQ 1570f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams ret = iop_adma_pq_zero_sum_self_test(adev); 1571f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams dev_dbg(&pdev->dev, "pq self test returned %d\n", ret); 1572f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams #else 1573f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams /* can not test raid6, so do not publish capability */ 1574f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams dma_cap_clear(DMA_PQ, dma_dev->cap_mask); 1575f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams dma_cap_clear(DMA_PQ_VAL, dma_dev->cap_mask); 1576f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams ret = 0; 1577f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams #endif 1578f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams if (ret) 1579f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams goto err_free_iop_chan; 1580f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams } 1581f6dbf651615900646fe0ba1ef5ce1027e5b4748dDan Williams 1582c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_printk(KERN_INFO, &pdev->dev, "Intel(R) IOP: " 15839308add6ea4fedeba37b0d7c4630a542bd34f214Dan Williams "( %s%s%s%s%s%s%s)\n", 1584b2f46fd8ef3dff2ab30f31126833f78b7480283aDan Williams dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "", 1585099f53cb50e45ef617a9f1d63ceec799e489418bDan Williams dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask) ? "pq_val " : "", 1586c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "", 1587099f53cb50e45ef617a9f1d63ceec799e489418bDan Williams dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask) ? "xor_val " : "", 1588c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_has_cap(DMA_MEMSET, dma_dev->cap_mask) ? "fill " : "", 1589c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "", 1590c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : ""); 1591c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1592c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_async_device_register(dma_dev); 1593c211092313b90f898dec61f35207fc282d1eadc3Dan Williams goto out; 1594c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1595c211092313b90f898dec61f35207fc282d1eadc3Dan Williams err_free_iop_chan: 1596c211092313b90f898dec61f35207fc282d1eadc3Dan Williams kfree(iop_chan); 1597c211092313b90f898dec61f35207fc282d1eadc3Dan Williams err_free_dma: 1598c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_free_coherent(&adev->pdev->dev, plat_data->pool_size, 1599c211092313b90f898dec61f35207fc282d1eadc3Dan Williams adev->dma_desc_pool_virt, adev->dma_desc_pool); 1600c211092313b90f898dec61f35207fc282d1eadc3Dan Williams err_free_adev: 1601c211092313b90f898dec61f35207fc282d1eadc3Dan Williams kfree(adev); 1602c211092313b90f898dec61f35207fc282d1eadc3Dan Williams out: 1603c211092313b90f898dec61f35207fc282d1eadc3Dan Williams return ret; 1604c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 1605c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1606c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan) 1607c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 1608c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_desc_slot *sw_desc, *grp_start; 1609c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_cookie_t cookie; 1610c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int slot_cnt, slots_per_op; 1611c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 16123d9b525b69bc3302d8355e5f5cf081a856c211e0Harvey Harrison dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); 1613c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1614c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_lock_bh(&iop_chan->lock); 1615c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot_cnt = iop_chan_memcpy_slot_count(0, &slots_per_op); 1616c211092313b90f898dec61f35207fc282d1eadc3Dan Williams sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 1617c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (sw_desc) { 1618c211092313b90f898dec61f35207fc282d1eadc3Dan Williams grp_start = sw_desc->group_head; 1619c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1620308136d1abcb2d759bac40ed4f5d42ac4af59d8bDan Williams list_splice_init(&sw_desc->tx_list, &iop_chan->chain); 1621636bdeaa1243327501edfd2a597ed7443eb4239aDan Williams async_tx_ack(&sw_desc->async_tx); 1622c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_desc_init_memcpy(grp_start, 0); 1623c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_desc_set_byte_count(grp_start, iop_chan, 0); 1624c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_desc_set_dest_addr(grp_start, iop_chan, 0); 1625c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_desc_set_memcpy_src_addr(grp_start, 0); 1626c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 16272a926e46022ad7a03e0ac167d8c2b0d88c12c5a8Russell King - ARM Linux cookie = dma_cookie_assign(&sw_desc->async_tx); 1628c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1629c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* initialize the completed cookie to be less than 1630c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * the most recently used cookie 1631c211092313b90f898dec61f35207fc282d1eadc3Dan Williams */ 16324d4e58de32a192fea65ab84509d17d199bd291c8Russell King - ARM Linux iop_chan->common.completed_cookie = cookie - 1; 1633c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1634c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* channel should not be busy */ 1635c211092313b90f898dec61f35207fc282d1eadc3Dan Williams BUG_ON(iop_chan_is_busy(iop_chan)); 1636c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1637c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* clear any prior error-status bits */ 1638c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_adma_device_clear_err_status(iop_chan); 1639c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1640c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* disable operation */ 1641c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan_disable(iop_chan); 1642c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1643c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* set the descriptor address */ 1644c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys); 1645c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1646c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* 1/ don't add pre-chained descriptors 1647c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * 2/ dummy read to flush next_desc write 1648c211092313b90f898dec61f35207fc282d1eadc3Dan Williams */ 1649c211092313b90f898dec61f35207fc282d1eadc3Dan Williams BUG_ON(iop_desc_get_next_desc(sw_desc)); 1650c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1651c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* run the descriptor */ 1652c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan_enable(iop_chan); 1653c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } else 1654c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_printk(KERN_ERR, iop_chan->device->common.dev, 1655c211092313b90f898dec61f35207fc282d1eadc3Dan Williams "failed to allocate null descriptor\n"); 1656c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_unlock_bh(&iop_chan->lock); 1657c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 1658c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1659c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan) 1660c211092313b90f898dec61f35207fc282d1eadc3Dan Williams{ 1661c211092313b90f898dec61f35207fc282d1eadc3Dan Williams struct iop_adma_desc_slot *sw_desc, *grp_start; 1662c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dma_cookie_t cookie; 1663c211092313b90f898dec61f35207fc282d1eadc3Dan Williams int slot_cnt, slots_per_op; 1664c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 16653d9b525b69bc3302d8355e5f5cf081a856c211e0Harvey Harrison dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); 1666c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1667c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_lock_bh(&iop_chan->lock); 1668c211092313b90f898dec61f35207fc282d1eadc3Dan Williams slot_cnt = iop_chan_xor_slot_count(0, 2, &slots_per_op); 1669c211092313b90f898dec61f35207fc282d1eadc3Dan Williams sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 1670c211092313b90f898dec61f35207fc282d1eadc3Dan Williams if (sw_desc) { 1671c211092313b90f898dec61f35207fc282d1eadc3Dan Williams grp_start = sw_desc->group_head; 1672308136d1abcb2d759bac40ed4f5d42ac4af59d8bDan Williams list_splice_init(&sw_desc->tx_list, &iop_chan->chain); 1673636bdeaa1243327501edfd2a597ed7443eb4239aDan Williams async_tx_ack(&sw_desc->async_tx); 1674c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_desc_init_null_xor(grp_start, 2, 0); 1675c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_desc_set_byte_count(grp_start, iop_chan, 0); 1676c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_desc_set_dest_addr(grp_start, iop_chan, 0); 1677c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_desc_set_xor_src_addr(grp_start, 0, 0); 1678c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_desc_set_xor_src_addr(grp_start, 1, 0); 1679c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 16802a926e46022ad7a03e0ac167d8c2b0d88c12c5a8Russell King - ARM Linux cookie = dma_cookie_assign(&sw_desc->async_tx); 1681c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1682c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* initialize the completed cookie to be less than 1683c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * the most recently used cookie 1684c211092313b90f898dec61f35207fc282d1eadc3Dan Williams */ 16854d4e58de32a192fea65ab84509d17d199bd291c8Russell King - ARM Linux iop_chan->common.completed_cookie = cookie - 1; 1686c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1687c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* channel should not be busy */ 1688c211092313b90f898dec61f35207fc282d1eadc3Dan Williams BUG_ON(iop_chan_is_busy(iop_chan)); 1689c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1690c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* clear any prior error-status bits */ 1691c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_adma_device_clear_err_status(iop_chan); 1692c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1693c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* disable operation */ 1694c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan_disable(iop_chan); 1695c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1696c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* set the descriptor address */ 1697c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys); 1698c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1699c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* 1/ don't add pre-chained descriptors 1700c211092313b90f898dec61f35207fc282d1eadc3Dan Williams * 2/ dummy read to flush next_desc write 1701c211092313b90f898dec61f35207fc282d1eadc3Dan Williams */ 1702c211092313b90f898dec61f35207fc282d1eadc3Dan Williams BUG_ON(iop_desc_get_next_desc(sw_desc)); 1703c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1704c211092313b90f898dec61f35207fc282d1eadc3Dan Williams /* run the descriptor */ 1705c211092313b90f898dec61f35207fc282d1eadc3Dan Williams iop_chan_enable(iop_chan); 1706c211092313b90f898dec61f35207fc282d1eadc3Dan Williams } else 1707c211092313b90f898dec61f35207fc282d1eadc3Dan Williams dev_printk(KERN_ERR, iop_chan->device->common.dev, 1708c211092313b90f898dec61f35207fc282d1eadc3Dan Williams "failed to allocate null descriptor\n"); 1709c211092313b90f898dec61f35207fc282d1eadc3Dan Williams spin_unlock_bh(&iop_chan->lock); 1710c211092313b90f898dec61f35207fc282d1eadc3Dan Williams} 1711c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1712c211092313b90f898dec61f35207fc282d1eadc3Dan Williamsstatic struct platform_driver iop_adma_driver = { 1713c211092313b90f898dec61f35207fc282d1eadc3Dan Williams .probe = iop_adma_probe, 1714bdf602bd737eb07d63d6fa2da826b4751fdf9babRussell King .remove = __devexit_p(iop_adma_remove), 1715c211092313b90f898dec61f35207fc282d1eadc3Dan Williams .driver = { 1716c211092313b90f898dec61f35207fc282d1eadc3Dan Williams .owner = THIS_MODULE, 1717c211092313b90f898dec61f35207fc282d1eadc3Dan Williams .name = "iop-adma", 1718c211092313b90f898dec61f35207fc282d1eadc3Dan Williams }, 1719c211092313b90f898dec61f35207fc282d1eadc3Dan Williams}; 1720c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1721c94e910535be72f0c6ac0c69e6acd8d44414e80dAxel Linmodule_platform_driver(iop_adma_driver); 1722c211092313b90f898dec61f35207fc282d1eadc3Dan Williams 1723c211092313b90f898dec61f35207fc282d1eadc3Dan WilliamsMODULE_AUTHOR("Intel Corporation"); 1724c211092313b90f898dec61f35207fc282d1eadc3Dan WilliamsMODULE_DESCRIPTION("IOP ADMA Engine Driver"); 1725c211092313b90f898dec61f35207fc282d1eadc3Dan WilliamsMODULE_LICENSE("GPL"); 1726c94e910535be72f0c6ac0c69e6acd8d44414e80dAxel LinMODULE_ALIAS("platform:iop-adma"); 1727