adma.c revision d2ebfb335b0426deb1a4fb14e4e926d81ecd8235
112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* 212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Copyright (C) 2006-2009 DENX Software Engineering. 312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Author: Yuri Tikhonov <yur@emcraft.com> 512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Further porting to arch/powerpc by 712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Anatolij Gustschin <agust@denx.de> 812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * This program is free software; you can redistribute it and/or modify it 1012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * under the terms of the GNU General Public License as published by the Free 1112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Software Foundation; either version 2 of the License, or (at your option) 1212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * any later version. 1312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 1412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * This program is distributed in the hope that it will be useful, but WITHOUT 1512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 1612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 1712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * more details. 1812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 1912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * You should have received a copy of the GNU General Public License along with 2012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * this program; if not, write to the Free Software Foundation, Inc., 59 2112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Temple Place - Suite 330, Boston, MA 02111-1307, USA. 2212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 2312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * The full GNU General Public License is included in this distribution in the 2412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * file called COPYING. 2512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 2612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 2712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* 2812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * This driver supports the asynchrounous DMA copy and RAID engines available 2912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * on the AMCC PPC440SPe Processors. 3012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) 3112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ADMA driver written by D.Williams. 3212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 3312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 3412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/init.h> 3512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/module.h> 3612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/async_tx.h> 3712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/delay.h> 3812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/dma-mapping.h> 3912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/spinlock.h> 4012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/interrupt.h> 415a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 4212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/uaccess.h> 4312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/proc_fs.h> 4412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/of.h> 4512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/of_platform.h> 4612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <asm/dcr.h> 4712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <asm/dcr-regs.h> 4812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include "adma.h" 49d2ebfb335b0426deb1a4fb14e4e926d81ecd8235Russell King - ARM Linux#include "../dmaengine.h" 5012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 5112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinenum ppc_adma_init_code { 5212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_OK = 0, 5312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_MEMRES, 5412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_MEMREG, 5512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_ALLOC, 5612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_COHERENT, 5712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_CHANNEL, 5812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_IRQ1, 5912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_IRQ2, 6012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_REGISTER 6112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 6212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 6312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic char *ppc_adma_errors[] = { 6412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_OK] = "ok", 6512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource", 6612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_MEMREG] = "failed to request memory region", 6712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev " 6812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "structure", 6912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for " 7012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "hardware descriptors", 7112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel", 7212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_IRQ1] = "failed to request first irq", 7312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_IRQ2] = "failed to request second irq", 7412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device", 7512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 7612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 7712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic enum ppc_adma_init_code 7812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM]; 7912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 8012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstruct ppc_dma_chan_ref { 8112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan; 8212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct list_head node; 8312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 8412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 8512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* The list of channels exported by ppc440spe ADMA */ 8612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstruct list_head 8712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list); 8812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 8912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* This flag is set when want to refetch the xor chain in the interrupt 9012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * handler 9112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 9212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 do_xor_refetch; 9312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 9412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* Pointer to DMA0, DMA1 CP/CS FIFO */ 9512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void *ppc440spe_dma_fifo_buf; 9612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 9712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* Pointers to last submitted to DMA0, DMA1 CDBs */ 9812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *chan_last_sub[3]; 9912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *chan_first_cdb[3]; 10012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 10112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* Pointer to last linked and submitted xor CB */ 10212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *xor_last_linked; 10312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *xor_last_submit; 10412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 10512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* This array is used in data-check operations for storing a pattern */ 10612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic char ppc440spe_qword[16]; 10712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 10812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic atomic_t ppc440spe_adma_err_irq_ref; 10912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic dcr_host_t ppc440spe_mq_dcr_host; 11012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic unsigned int ppc440spe_mq_dcr_len; 11112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 11212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* Since RXOR operations use the common register (MQ0_CF2H) for setting-up 11312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the block size in transactions, then we do not allow to activate more than 11412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * only one RXOR transactions simultaneously. So use this var to store 11512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is 11612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * set) or not (PPC440SPE_RXOR_RUN is clear). 11712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 11812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic unsigned long ppc440spe_rxor_state; 11912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 12012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* These are used in enable & check routines 12112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 12212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 ppc440spe_r6_enabled; 12312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_chan *ppc440spe_r6_tchan; 12412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct completion ppc440spe_r6_test_comp; 12512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 12612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_dma2rxor_prep_src( 12712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 12812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_rxor *cursor, int index, 12912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, u32 addr); 13012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_set_src( 13112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 13212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index, dma_addr_t addr); 13312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_set_mult( 13412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 13512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index, u8 mult); 13612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 13712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#ifdef ADMA_LL_DEBUG 13812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define ADMA_LL_DBG(x) ({ if (1) x; 0; }) 13912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#else 14012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define ADMA_LL_DBG(x) ({ if (0) x; 0; }) 14112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#endif 14212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 14312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void print_cb(struct ppc440spe_adma_chan *chan, void *block) 14412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 14512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *cdb; 14612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *cb; 14712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 14812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 14912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 15012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 0: 15112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 1: 15212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb = block; 15312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 15412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("CDB at %p [%d]:\n" 15512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n" 15612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t sg1u 0x%08x sg1l 0x%08x\n" 15712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t sg2u 0x%08x sg2l 0x%08x\n" 15812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t sg3u 0x%08x sg3l 0x%08x\n", 15912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb, chan->device->id, 16012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt), 16112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l), 16212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l), 16312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l) 16412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ); 16512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 16612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 2: 16712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb = block; 16812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 16912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("CB at %p [%d]:\n" 17012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n" 17112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t cbtah 0x%08x cbtal 0x%08x\n" 17212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t cblah 0x%08x cblal 0x%08x\n", 17312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb, chan->device->id, 17412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb->cbc, cb->cbbc, cb->cbs, 17512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb->cbtah, cb->cbtal, 17612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb->cblah, cb->cblal); 17712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < 16; i++) { 17812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i && !cb->ops[i].h && !cb->ops[i].l) 17912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 18012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n", 18112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i, cb->ops[i].h, cb->ops[i].l); 18212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 18312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 18412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 18512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 18612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 18712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void print_cb_list(struct ppc440spe_adma_chan *chan, 18812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter) 18912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 19012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (; iter; iter = iter->hw_next) 19112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin print_cb(chan, iter->hw_desc); 19212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 19312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 19412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src, 19512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt) 19612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 19712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 19812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 19912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\n%s(%d):\nsrc: ", __func__, id); 20012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 20112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx ", src[i]); 20212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("dst:\n\t0x%016llx\n", dst); 20312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 20412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 20512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src, 20612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt) 20712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 20812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 20912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 21012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\n%s(%d):\nsrc: ", __func__, id); 21112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 21212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx ", src[i]); 21312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("dst: "); 21412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < 2; i++) 21512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx ", dst[i]); 21612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 21712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 21812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src, 21912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt, 22012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf) 22112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 22212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 22312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 22412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id); 22512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (scf) { 22612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 22712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]); 22812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 22912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 23012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx(no) ", src[i]); 23112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 23212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 23312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("dst: "); 23412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < 2; i++) 23512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx ", src[src_cnt + i]); 23612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 23712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 23812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/****************************************************************************** 23912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Command (Descriptor) Blocks low-level routines 24012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ******************************************************************************/ 24112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 24212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT 24312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * pseudo operation 24412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 24512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc, 24612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 24712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 24812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *p; 24912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 25012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 25112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 25212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin p = desc->hw_desc; 25312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 25412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* NOP with Command Block Complete Enable */ 25512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin p->cbc = XOR_CBCR_CBCE_BIT; 25612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 25712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 25812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 25912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 26012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* NOP with interrupt */ 26112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &desc->flags); 26212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 26312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 26412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id, 26512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 26612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 26712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 26812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 26912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 27012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 27112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR 27212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * pseudo operation 27312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 27412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc) 27512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 27612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 27712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 27812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = 0; 27912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = 1; 28012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 28112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 28212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 28312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation 28412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 28512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc, 28612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, unsigned long flags) 28712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 28812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *hw_desc = desc->hw_desc; 28912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 29012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 29112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 29212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = src_cnt; 29312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = 1; 29412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 29512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt; 29612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 29712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Enable interrupt on completion */ 29812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc |= XOR_CBCR_CBCE_BIT; 29912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 30012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 30112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 30212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ 30312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * operation in DMA2 controller 30412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 30512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc, 30612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int dst_cnt, int src_cnt, unsigned long flags) 30712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 30812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *hw_desc = desc->hw_desc; 30912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 31012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 31112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 31212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = src_cnt; 31312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = dst_cnt; 31412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags)); 31512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->descs_per_op = 0; 31612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 31712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc = XOR_CBCR_TGT_BIT; 31812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 31912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Enable interrupt on completion */ 32012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc |= XOR_CBCR_CBCE_BIT; 32112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 32212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 32312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define DMA_CTRL_FLAGS_LAST DMA_PREP_FENCE 32412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define DMA_PREP_ZERO_P (DMA_CTRL_FLAGS_LAST << 1) 32512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define DMA_PREP_ZERO_Q (DMA_PREP_ZERO_P << 1) 32612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 32712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 32812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation 32912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * with DMA0/1 33012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 33112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc, 33212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int dst_cnt, int src_cnt, unsigned long flags, 33312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op) 33412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 33512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 33612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 33712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 dopc; 33812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 33912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Common initialization of a PQ descriptors chain */ 34012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bits(op, &desc->flags); 34112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = src_cnt; 34212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = dst_cnt; 34312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 34412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR MULTICAST if both P and Q are being computed 34512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * MV_SG1_SG2 if Q only 34612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 34712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ? 34812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2; 34912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 35012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &desc->group_list, chain_node) { 35112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 35212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 35312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 35412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(!list_is_last(&iter->chain_node, 35512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &desc->group_list))) { 35612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 35712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 35812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 35912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 36012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 36112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is the last descriptor. 36212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * this slot will be pasted from ADMA level 36312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * each time it wants to configure parameters 36412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * of the transaction (src, dst, ...) 36512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 36612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 36712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 36812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 36912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 37012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 37112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 37212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 37312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 37412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set OPS depending on WXOR/RXOR type of operation */ 37512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) { 37612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is a WXOR only chain: 37712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - first descriptors are for zeroing destinations 37812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * if PPC440SPE_ZERO_P/Q set; 37912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - descriptors remained are for GF-XOR operations. 38012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 38112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&desc->group_list, 38212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 38312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 38412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 38512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) { 38612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 38712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 38812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 38912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 39012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 39112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 39212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 39312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) { 39412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 39512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 39612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 39712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 39812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 39912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 40012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 40112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &desc->group_list, chain_node) { 40212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 40312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = dopc; 40412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 40512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 40612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is either RXOR-only or mixed RXOR/WXOR */ 40712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 40812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* The first 1 or 2 slots in chain are always RXOR, 40912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * if need to calculate P & Q, then there are two 41012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * RXOR slots; if only P or only Q, then there is one 41112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 41212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&desc->group_list, 41312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 41412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 41512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 41612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 41712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 41812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (desc->dst_cnt == DMA_DEST_MAX_NUM) { 41912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 42012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 42112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 42212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 42312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 42412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 42512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 42612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* The remaining descs (if any) are WXORs */ 42712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) { 42812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 42912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 43012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 43112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &desc->group_list, 43212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 43312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 43412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = dopc; 43512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 43612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 43712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 43812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 43912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 44012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 44112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor 44212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for PQ_ZERO_SUM operation 44312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 44412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_dma01pqzero_sum( 44512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 44612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int dst_cnt, int src_cnt) 44712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 44812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 44912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 45012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i = 0; 45112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST : 45212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_OPC_MV_SG1_SG2; 45312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 45412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Initialize starting from 2nd or 3rd descriptor dependent 45512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * on dst_cnt. First one or two slots are for cloning P 45612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * and/or Q to chan->pdest and/or chan->qdest as we have 45712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * to preserve original P/Q. 45812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 45912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&desc->group_list, 46012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 46112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 46212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 46312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 46412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dst_cnt > 1) { 46512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 46612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 46712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 46812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* initialize each source descriptor in chain */ 46912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &desc->group_list, chain_node) { 47012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 47112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 47212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->src_cnt = 0; 47312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->dst_cnt = 0; 47412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 47512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is a ZERO_SUM operation: 47612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - <src_cnt> descriptors starting from 2nd or 3rd 47712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor are for GF-XOR operations; 47812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - remaining <dst_cnt> descriptors are for checking the result 47912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 48012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i++ < src_cnt) 48112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* MV_SG1_SG2 if only Q is being verified 48212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * MULTICAST if both P and Q are being verified 48312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 48412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = dopc; 48512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 48612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA_CDB_OPC_DCHECK128 operation */ 48712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_DCHECK128; 48812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 48912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(!list_is_last(&iter->chain_node, 49012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &desc->group_list))) { 49112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 49212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 49312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 49412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 49512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 49612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is the last descriptor. 49712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * this slot will be pasted from ADMA level 49812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * each time it wants to configure parameters 49912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * of the transaction (src, dst, ...) 50012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 50112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 50212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* always enable interrupt generation since we get 50312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the status of pqzero from the handler 50412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 50512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 50612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 50712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 50812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = src_cnt; 50912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = dst_cnt; 51012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 51112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 51212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 51312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation 51412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 51512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc, 51612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long flags) 51712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 51812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc = desc->hw_desc; 51912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 52012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 52112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 52212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = 1; 52312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = 1; 52412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 52512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 52612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &desc->flags); 52712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 52812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &desc->flags); 52912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 53012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 53112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 53212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 53312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 53412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_memset - initialize the descriptor for MEMSET operation 53512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 53612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_memset(struct ppc440spe_adma_desc_slot *desc, 53712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int value, unsigned long flags) 53812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 53912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc = desc->hw_desc; 54012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 54112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 54212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 54312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = 1; 54412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = 1; 54512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 54612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 54712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &desc->flags); 54812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 54912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &desc->flags); 55012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 55112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->sg1u = hw_desc->sg1l = cpu_to_le32((u32)value); 55212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->sg3u = hw_desc->sg3l = cpu_to_le32((u32)value); 55312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_DFILL128; 55412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 55512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 55612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 55712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_src_addr - set source address into the descriptor 55812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 55912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc, 56012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 56112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_idx, dma_addr_t addrh, 56212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addrl) 56312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 56412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 56512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 56612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin phys_addr_t addr64, tmplow, tmphi; 56712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 56812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 56912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 57012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 57112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addrh) { 57212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr64 = addrl; 57312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmphi = (addr64 >> 32); 57412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmplow = (addr64 & 0xFFFFFFFF); 57512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 57612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmphi = addrh; 57712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmplow = addrl; 57812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 57912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 58012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow); 58112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi); 58212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 58312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 58412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 58512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->ops[src_idx].l = addrl; 58612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->ops[src_idx].h |= addrh; 58712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 58812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 58912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 59012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 59112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 59212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_src_mult - set source address mult into the descriptor 59312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 59412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc, 59512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, u32 mult_index, 59612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int sg_index, unsigned char mult_value) 59712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 59812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 59912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 60012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 *psgu; 60112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 60212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 60312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 60412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 60512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 60612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 60712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (sg_index) { 60812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* for RXOR operations set multiplier 60912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * into source cued address 61012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 61112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_SG_SRC: 61212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgu = &dma_hw_desc->sg1u; 61312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 61412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* for WXOR operations set multiplier 61512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * into destination cued address(es) 61612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 61712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_SG_DST1: 61812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgu = &dma_hw_desc->sg2u; 61912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 62012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_SG_DST2: 62112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgu = &dma_hw_desc->sg3u; 62212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 62312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 62412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 62512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 62612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 62712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *psgu |= cpu_to_le32(mult_value << mult_index); 62812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 62912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 63012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 63112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 63212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 63312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 63412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 63512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 63612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 63712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 63812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_dest_addr - set destination address into the descriptor 63912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 64012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc, 64112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 64212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addrh, dma_addr_t addrl, 64312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 dst_idx) 64412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 64512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 64612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 64712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin phys_addr_t addr64, tmphi, tmplow; 64812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 *psgu, *psgl; 64912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 65012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 65112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 65212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 65312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addrh) { 65412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr64 = addrl; 65512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmphi = (addr64 >> 32); 65612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmplow = (addr64 & 0xFFFFFFFF); 65712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 65812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmphi = addrh; 65912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmplow = addrl; 66012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 66112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 66212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 66312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u; 66412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l; 66512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 66612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *psgl = cpu_to_le32((u32)tmplow); 66712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *psgu |= cpu_to_le32((u32)tmphi); 66812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 66912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 67012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 67112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbtal = addrl; 67212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbtah |= addrh; 67312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 67412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 67512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 67612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 67712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 67812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_byte_count - set number of data bytes involved 67912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * into the operation 68012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 68112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc, 68212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 68312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 byte_count) 68412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 68512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 68612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 68712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 68812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 68912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 69012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 69112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 69212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->cnt = cpu_to_le32(byte_count); 69312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 69412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 69512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 69612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbbc = byte_count; 69712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 69812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 69912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 70012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 70112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 70212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_rxor_block_size - set RXOR block size 70312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 70412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count) 70512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 70612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* assume that byte_count is aligned on the 512-boundary; 70712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * thus write it directly to the register (bits 23:31 are 70812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * reserved there). 70912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 71012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count); 71112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 71212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 71312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 71412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_dcheck - set CHECK pattern 71512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 71612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc, 71712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, u8 *qword) 71812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 71912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 72012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 72112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 72212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 72312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 72412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 72512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(qword[0], &dma_hw_desc->sg3l); 72612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(qword[4], &dma_hw_desc->sg3u); 72712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(qword[8], &dma_hw_desc->sg2l); 72812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(qword[12], &dma_hw_desc->sg2u); 72912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 73012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 73112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 73212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 73312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 73412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 73512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 73612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_xor_set_link - set link address in xor CB 73712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 73812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc, 73912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *next_desc) 74012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 74112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc = prev_desc->hw_desc; 74212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 74312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!next_desc || !(next_desc->phys))) { 74412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n", 74512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, next_desc, 74612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin next_desc ? next_desc->phys : 0); 74712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 74812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 74912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 75012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbs = 0; 75112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cblal = next_desc->phys; 75212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cblah = 0; 75312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT; 75412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 75512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 75612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 75712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_link - set the address of descriptor following this 75812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor in chain 75912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 76012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan, 76112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *prev_desc, 76212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *next_desc) 76312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 76412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long flags; 76512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *tail = next_desc; 76612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 76712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!prev_desc || !next_desc || 76812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (prev_desc->hw_next && prev_desc->hw_next != next_desc))) { 76912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* If previous next is overwritten something is wrong. 77012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * though we may refetch from append to initiate list 77112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * processing; in this case - it's ok. 77212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 77312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; " 77412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "prev->hw_next=0x%p\n", __func__, prev_desc, 77512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin next_desc, prev_desc ? prev_desc->hw_next : 0); 77612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 77712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 77812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 77912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin local_irq_save(flags); 78012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 78112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do s/w chaining both for DMA and XOR descriptors */ 78212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prev_desc->hw_next = next_desc; 78312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 78412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 78512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 78612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 78712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 78812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 78912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* bind descriptor to the chain */ 79012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (tail->hw_next) 79112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tail = tail->hw_next; 79212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_linked = tail; 79312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 79412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (prev_desc == xor_last_submit) 79512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do not link to the last submitted CB */ 79612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 79712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_xor_set_link(prev_desc, next_desc); 79812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 79912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 80012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 80112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin local_irq_restore(flags); 80212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 80312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 80412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 80512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_get_src_addr - extract the source address from the descriptor 80612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 80712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 ppc440spe_desc_get_src_addr(struct ppc440spe_adma_desc_slot *desc, 80812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, int src_idx) 80912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 81012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 81112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 81212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 81312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 81412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 81512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 81612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 81712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* May have 0, 1, 2, or 3 sources */ 81812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (dma_hw_desc->opc) { 81912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_NO_OP: 82012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DFILL128: 82112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 82212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DCHECK128: 82312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(src_idx)) { 82412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: try to get %d source for" 82512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " DCHECK128\n", __func__, src_idx); 82612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 82712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 82812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu(dma_hw_desc->sg1l); 82912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MULTICAST: 83012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MV_SG1_SG2: 83112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(src_idx > 2)) { 83212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: try to get %d source from" 83312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " DMA descr\n", __func__, src_idx); 83412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 83512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 83612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_idx) { 83712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (le32_to_cpu(dma_hw_desc->sg1u) & 83812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_WIN_MSK) { 83912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 region; 84012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 84112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_idx == 1) 84212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu( 84312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1l) + 84412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->unmap_len; 84512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 84612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin region = (le32_to_cpu( 84712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1u)) >> 84812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 84912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 85012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin region &= DMA_CUED_REGION_MSK; 85112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (region) { 85212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_RXOR123: 85312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu( 85412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1l) + 85512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (desc->unmap_len << 1); 85612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_RXOR124: 85712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu( 85812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1l) + 85912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (desc->unmap_len * 3); 86012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_RXOR125: 86112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu( 86212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1l) + 86312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (desc->unmap_len << 2); 86412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 86512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR 86612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "%s: try to" 86712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " get src3 for region %02x" 86812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "PPC440SPE_DESC_RXOR12?\n", 86912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, region); 87012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 87112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 87212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 87312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR 87412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "%s: try to get %d" 87512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " source for non-cued descr\n", 87612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, src_idx); 87712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 87812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 87912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 88012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu(dma_hw_desc->sg1l); 88112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 88212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: unknown OPC 0x%02x\n", 88312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dma_hw_desc->opc); 88412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 88512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 88612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu(dma_hw_desc->sg1l); 88712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 88812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* May have up to 16 sources */ 88912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 89012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return xor_hw_desc->ops[src_idx].l; 89112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 89212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 89312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 89412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 89512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 89612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_get_dest_addr - extract the destination address from the 89712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor 89812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 89912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 ppc440spe_desc_get_dest_addr(struct ppc440spe_adma_desc_slot *desc, 90012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, int idx) 90112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 90212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 90312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 90412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 90512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 90612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 90712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 90812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 90912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 91012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(!idx)) 91112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu(dma_hw_desc->sg2l); 91212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu(dma_hw_desc->sg3l); 91312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 91412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 91512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return xor_hw_desc->cbtal; 91612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 91712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 91812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 91912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 92012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 92112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_get_src_num - extract the number of source addresses from 92212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the descriptor 92312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 92412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 ppc440spe_desc_get_src_num(struct ppc440spe_adma_desc_slot *desc, 92512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 92612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 92712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 92812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 92912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 93012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 93112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 93212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 93312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 93412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 93512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (dma_hw_desc->opc) { 93612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_NO_OP: 93712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DFILL128: 93812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 93912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DCHECK128: 94012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 94112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MV_SG1_SG2: 94212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MULTICAST: 94312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 94412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Only for RXOR operations we have more than 94512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * one source 94612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 94712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (le32_to_cpu(dma_hw_desc->sg1u) & 94812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_WIN_MSK) { 94912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR op, there are 2 or 3 sources */ 95012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (((le32_to_cpu(dma_hw_desc->sg1u) >> 95112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF) & 95212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_MSK) == DMA_RXOR12) { 95312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR 1-2 */ 95412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 2; 95512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 95612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR 1-2-3/1-2-4/1-2-5 */ 95712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 3; 95812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 95912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 96012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 96112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 96212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: unknown OPC 0x%02x\n", 96312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dma_hw_desc->opc); 96412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 96512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 96612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 96712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* up to 16 sources */ 96812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 96912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return xor_hw_desc->cbc & XOR_CDCR_OAC_MSK; 97012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 97112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 97212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 97312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 97412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 97512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 97612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 97712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_get_dst_num - get the number of destination addresses in 97812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * this descriptor 97912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 98012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 ppc440spe_desc_get_dst_num(struct ppc440spe_adma_desc_slot *desc, 98112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 98212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 98312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 98412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 98512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 98612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 98712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 98812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* May be 1 or 2 destinations */ 98912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 99012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (dma_hw_desc->opc) { 99112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_NO_OP: 99212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DCHECK128: 99312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 99412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MV_SG1_SG2: 99512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DFILL128: 99612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 99712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MULTICAST: 99812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (desc->dst_cnt == 2) 99912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 2; 100012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 100112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 100212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 100312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: unknown OPC 0x%02x\n", 100412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dma_hw_desc->opc); 100512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 100612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 100712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 100812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Always only 1 destination */ 100912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 101012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 101112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 101212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 101312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 101412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 101512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 101612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 101712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_get_link - get the address of the descriptor that 101812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * follows this one 101912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 102012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc, 102112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 102212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 102312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!desc->hw_next) 102412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 102512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 102612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return desc->hw_next->phys; 102712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 102812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 102912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 103012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_is_aligned - check alignment 103112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 103212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline int ppc440spe_desc_is_aligned( 103312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, int num_slots) 103412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 103512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return (desc->idx & (num_slots - 1)) ? 0 : 1; 103612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 103712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 103812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 103912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_xor_slot_count - get the number of slots necessary for 104012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * XOR operation 104112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 104212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt, 104312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int *slots_per_op) 104412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 104512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 104612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 104712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* each XOR descriptor provides up to 16 source operands */ 104812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS; 104912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 105012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT)) 105112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return slot_cnt; 105212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 105312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: len %d > max %d !!\n", 105412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 105512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 105612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return slot_cnt; 105712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 105812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 105912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 106012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for 106112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * DMA2 PQ operation 106212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 106312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs, 106412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, size_t len) 106512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 106612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin signed long long order = 0; 106712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int state = 0; 106812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int addr_count = 0; 106912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 107012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 1; i < src_cnt; i++) { 107112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t cur_addr = srcs[i]; 107212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t old_addr = srcs[i-1]; 107312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (state) { 107412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 0: 107512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cur_addr == old_addr + len) { 107612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct RXOR */ 107712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 1; 107812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 1; 107912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 108012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 108112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (old_addr == cur_addr + len) { 108212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse RXOR */ 108312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = -1; 108412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 1; 108512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 108612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 108712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 108812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 3; 108912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 109012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 109112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 1: 109212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-2 || (order == -1 109312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin && cur_addr != old_addr - len)) { 109412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 109512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 109612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 109712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cur_addr == old_addr + len*order) { 109812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 2; 109912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 110012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 110112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cur_addr == old_addr + 2*len) { 110212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 2; 110312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 110412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 110512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cur_addr == old_addr + 3*len) { 110612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 2; 110712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 110812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 110912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 111012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 111112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 111212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 111312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 111412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 111512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 2: 111612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 111712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 111812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 111912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 112012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 112112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (state == 3) 112212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 112312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 112412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt <= 1 || (state != 1 && state != 2)) { 112512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n", 112612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, src_cnt, state, addr_count, order); 112712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 112812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("\t[%d] 0x%llx \n", i, srcs[i]); 112912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 113012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 113112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 113212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS; 113312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 113412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 113512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 113612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/****************************************************************************** 113712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ADMA channel low-level routines 113812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ******************************************************************************/ 113912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 114012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 114112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan); 114212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan); 114312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 114412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 114512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine 114612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 114712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_device_clear_eot_status( 114812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 114912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 115012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_regs *dma_reg; 115112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 115212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 *p = chan->device->dma_desc_pool_virt; 115312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *cdb; 115412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 rv, i; 115512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 115612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 115712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 115812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 115912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* read FIFO to ack */ 116012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_reg = chan->device->dma_reg; 116112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while ((rv = ioread32(&dma_reg->csfpl))) { 116212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i = rv & DMA_CDB_ADDR_MSK; 116312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb = (struct dma_cdb *)&p[i - 116412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (u32)chan->device->dma_desc_pool]; 116512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 116612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Clear opcode to ack. This is necessary for 116712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ZeroSum operations only 116812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 116912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb->opc = 0; 117012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 117112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_RXOR_RUN, 117212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &ppc440spe_rxor_state)) { 117312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* probably this is a completed RXOR op, 117412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * get pointer to CDB using the fact that 117512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * physical and virtual addresses of CDB 117612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * in pools have the same offsets 117712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 117812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (le32_to_cpu(cdb->sg1u) & 117912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE) { 118012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is a RXOR */ 118112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_RXOR_RUN, 118212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &ppc440spe_rxor_state); 118312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 118412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 118512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 118612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rv & DMA_CDB_STATUS_MSK) { 118712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* ZeroSum check failed 118812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 118912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 119012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t phys = rv & ~DMA_CDB_MSK; 119112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 119212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 119312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Update the status of corresponding 119412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor. 119512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 119612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &chan->chain, 119712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 119812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->phys == phys) 119912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 120012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 120112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 120212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * if cannot find the corresponding 120312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * slot it's a bug 120412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 120512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(&iter->chain_node == &chan->chain); 120612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 120712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->xor_check_result) { 120812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_PCHECK, 120912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags)) { 121012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *iter->xor_check_result |= 121112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin SUM_CHECK_P_RESULT; 121212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 121312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_QCHECK, 121412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags)) { 121512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *iter->xor_check_result |= 121612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin SUM_CHECK_Q_RESULT; 121712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 121812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 121912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 122012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 122112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 122212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 122312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rv = ioread32(&dma_reg->dsts); 122412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rv) { 122512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("DMA%d err status: 0x%x\n", 122612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id, rv); 122712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* write back to clear */ 122812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(rv, &dma_reg->dsts); 122912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 123012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 123112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 123212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reset status bits to ack */ 123312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 123412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rv = ioread32be(&xor_reg->sr); 123512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(rv, &xor_reg->sr); 123612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 123712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) { 123812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rv & XOR_IE_RPTIE_BIT) { 123912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Read PLB Timeout Error. 124012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Try to resubmit the CB 124112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 124212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 val = ioread32be(&xor_reg->ccbalr); 124312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 124412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(val, &xor_reg->cblalr); 124512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 124612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin val = ioread32be(&xor_reg->crsr); 124712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(val | XOR_CRSR_XAE_BIT, 124812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &xor_reg->crsr); 124912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 125012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("XOR ERR 0x%x status\n", rv); 125112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 125212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 125312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 125412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* if the XORcore is idle, but there are unprocessed CBs 125512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * then refetch the s/w chain here 125612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 125712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) && 125812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin do_xor_refetch) 125912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_append(chan); 126012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 126112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 126212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 126312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 126412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 126512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_is_busy - get the channel status 126612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 126712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan) 126812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 126912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_regs *dma_reg; 127012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 127112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int busy = 0; 127212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 127312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 127412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 127512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 127612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_reg = chan->device->dma_reg; 127712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* if command FIFO's head and tail pointers are equal and 127812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * status tail is the same as command, then channel is free 127912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 128012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) || 128112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp)) 128212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin busy = 1; 128312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 128412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 128512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* use the special status bit for the XORcore 128612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 128712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 128812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0; 128912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 129012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 129112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 129212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return busy; 129312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 129412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 129512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 129612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_set_first_xor_descriptor - init XORcore chain 129712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 129812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_set_first_xor_descriptor( 129912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 130012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *next_desc) 130112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 130212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg = chan->device->xor_reg; 130312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 130412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) 130512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_INFO "%s: Warn: XORcore is running " 130612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "when try to set the first CDB!\n", 130712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 130812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 130912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_submit = xor_last_linked = next_desc; 131012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 131112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr); 131212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 131312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(next_desc->phys, &xor_reg->cblalr); 131412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(0, &xor_reg->cblahr); 131512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT, 131612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &xor_reg->cbcr); 131712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 131812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->hw_chain_inited = 1; 131912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 132012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 132112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 132212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO. 132312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * called with irqs disabled 132412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 132512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan, 132612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc) 132712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 132812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 pcdb; 132912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_regs *dma_reg = chan->device->dma_reg; 133012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 133112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pcdb = desc->phys; 133212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_INT, &desc->flags)) 133312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pcdb |= DMA_CDB_NO_INT; 133412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 133512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan_last_sub[chan->device->id] = desc; 133612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 133712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(print_cb(chan, desc->hw_desc)); 133812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 133912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(pcdb, &dma_reg->cpfpl); 134012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 134112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 134212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 134312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_append - update the h/w chain in the channel 134412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 134512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan) 134612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 134712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 134812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 134912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb; 135012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 cur_desc; 135112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long flags; 135212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 135312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin local_irq_save(flags); 135412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 135512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 135612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 135712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 135812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cur_desc = ppc440spe_chan_get_current_descriptor(chan); 135912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 136012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(cur_desc)) { 136112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = chan_last_sub[chan->device->id]; 136212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!iter); 136312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 136412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* first peer */ 136512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = chan_first_cdb[chan->device->id]; 136612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!iter); 136712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_dma_put_desc(chan, iter); 136812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->hw_chain_inited = 1; 136912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 137012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 137112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* is there something new to append */ 137212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!iter->hw_next) 137312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 137412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 137512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* flush descriptors from the s/w queue to fifo */ 137612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_continue(iter, &chan->chain, chain_node) { 137712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_dma_put_desc(chan, iter); 137812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!iter->hw_next) 137912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 138012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 138112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 138212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 138312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* update h/w links and refetch */ 138412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!xor_last_submit->hw_next) 138512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 138612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 138712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 138812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* the last linked CDB has to generate an interrupt 138912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * that we'd be able to append the next lists to h/w 139012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * regardless of the XOR engine state at the moment of 139112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * appending of these next lists 139212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 139312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb = xor_last_linked->hw_desc; 139412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->cbc |= XOR_CBCR_CBCE_BIT; 139512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 139612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) { 139712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* XORcore is idle. Refetch now */ 139812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin do_xor_refetch = 0; 139912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_xor_set_link(xor_last_submit, 140012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_submit->hw_next); 140112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 140212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(print_cb_list(chan, 140312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_submit->hw_next)); 140412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 140512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_submit = xor_last_linked; 140612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(ioread32be(&xor_reg->crsr) | 140712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT, 140812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &xor_reg->crsr); 140912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 141012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* XORcore is running. Refetch later in the handler */ 141112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin do_xor_refetch = 1; 141212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 141312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 141412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 141512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 141612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 141712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin local_irq_restore(flags); 141812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 141912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 142012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 142112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor 142212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 142312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 142412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan) 142512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 142612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_regs *dma_reg; 142712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 142812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 142912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!chan->hw_chain_inited)) 143012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* h/w descriptor chain is not initialized yet */ 143112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 143212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 143312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 143412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 143512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 143612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_reg = chan->device->dma_reg; 143712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK); 143812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 143912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 144012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ioread32be(&xor_reg->ccbalr); 144112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 144212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 144312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 144412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 144512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 144612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_run - enable the channel 144712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 144812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan) 144912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 145012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 145112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 145212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 145312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 145412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 145512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMAs are always enabled, do nothing */ 145612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 145712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 145812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* drain write buffer */ 145912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 146012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 146112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* fetch descriptor pointed to in <link> */ 146212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT, 146312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &xor_reg->crsr); 146412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 146512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 146612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 146712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 146812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/****************************************************************************** 146912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ADMA device level 147012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ******************************************************************************/ 147112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 147212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan); 147312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan); 147412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 147512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic dma_cookie_t 147612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx); 147712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 147812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx, 147912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index); 148012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 148112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx, 148212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index); 148312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 148412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 148512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx, 148612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *paddr, unsigned long flags); 148712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 148812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx, 148912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index); 149012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 149112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx, 149212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned char mult, int index, int dst_pos); 149312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 149412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx, 149512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t paddr, dma_addr_t qaddr); 149612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 149712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct page *ppc440spe_rxor_srcs[32]; 149812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 149912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 150012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_can_rxor - check if the operands may be processed with RXOR 150112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 150212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len) 150312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 150412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i, order = 0, state = 0; 150512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int idx = 0; 150612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 150712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!(src_cnt > 1))) 150812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 150912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 151012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs)); 151112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 151212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Skip holes in the source list before checking */ 151312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) { 151412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!srcs[i]) 151512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 151612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_srcs[idx++] = srcs[i]; 151712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 151812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt = idx; 151912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 152012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 1; i < src_cnt; i++) { 152112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin char *cur_addr = page_address(ppc440spe_rxor_srcs[i]); 152212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]); 152312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 152412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (state) { 152512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 0: 152612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cur_addr == old_addr + len) { 152712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct RXOR */ 152812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 1; 152912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 1; 153012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (old_addr == cur_addr + len) { 153112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse RXOR */ 153212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = -1; 153312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 1; 153412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 153512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out; 153612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 153712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 1: 153812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((i == src_cnt - 2) || 153912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (order == -1 && cur_addr != old_addr - len)) { 154012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 154112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 154212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if ((cur_addr == old_addr + len * order) || 154312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (cur_addr == old_addr + 2 * len) || 154412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (cur_addr == old_addr + 3 * len)) { 154512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 2; 154612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 154712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 154812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 154912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 155012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 155112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 2: 155212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 155312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 155412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 155512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 155612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 155712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 155812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout: 155912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (state == 1 || state == 2) 156012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 156112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 156212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 156312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 156412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 156512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 156612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_device_estimate - estimate the efficiency of processing 156712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the operation given on this channel. It's assumed that 'chan' is 156812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * capable to process 'cap' type of operation. 156912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @chan: channel to use 157012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @cap: type of transaction 157112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @dst_lst: array of destination pointers 157212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @dst_cnt: number of destination operands 157312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @src_lst: array of source pointers 157412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @src_cnt: number of source operands 157512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @src_sz: size of each source operand 157612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 157712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_estimate(struct dma_chan *chan, 157812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt, 157912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct page **src_lst, int src_cnt, size_t src_sz) 158012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 158112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int ef = 1; 158212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 158312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cap == DMA_PQ || cap == DMA_PQ_VAL) { 158412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* If RAID-6 capabilities were not activated don't try 158512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * to use them 158612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 158712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!ppc440spe_r6_enabled)) 158812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -1; 158912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 159012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* In the current implementation of ppc440spe ADMA driver it 159112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * makes sense to pick out only pq case, because it may be 159212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * processed: 159312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (1) either using Biskup method on DMA2; 159412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (2) or on DMA0/1. 159512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Thus we give a favour to (1) if the sources are suitable; 159612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * else let it be processed on one of the DMA0/1 engines. 159712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * In the sum_product case where destination is also the 159812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * source process it on DMA0/1 only. 159912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 160012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) { 160112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 160212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1]) 160312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ef = 0; /* sum_product case, process on DMA0/1 */ 160412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz)) 160512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ef = 3; /* override (DMA0/1 + idle) */ 160612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 160712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ef = 0; /* can't process on DMA2 if !rxor */ 160812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 160912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 161012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* channel idleness increases the priority */ 161112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(ef) && 161212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan))) 161312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ef++; 161412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 161512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ef; 161612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 161712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 161812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstruct dma_chan * 161912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap, 162012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct page **dst_lst, int dst_cnt, struct page **src_lst, 162112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, size_t src_sz) 162212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 162312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *best_chan = NULL; 162412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc_dma_chan_ref *ref; 162512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int best_rank = -1; 162612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 162712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!src_sz)) 162812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 162912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_sz > PAGE_SIZE) { 163012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 163112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * should a user of the api ever pass > PAGE_SIZE requests 163212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * we sort out cases where temporary page-sized buffers 163312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * are used. 163412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 163512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (cap) { 163612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_PQ: 163712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt == 1 && dst_lst[1] == src_lst[0]) 163812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 163912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt == 2 && dst_lst[1] == src_lst[1]) 164012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 164112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 164212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_PQ_VAL: 164312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_XOR_VAL: 164412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 164512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 164612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 164712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 164812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 164912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 165012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) { 165112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(cap, ref->chan->device->cap_mask)) { 165212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int rank; 165312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 165412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst, 165512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst_cnt, src_lst, src_cnt, src_sz); 165612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rank > best_rank) { 165712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin best_rank = rank; 165812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin best_chan = ref->chan; 165912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 166012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 166112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 166212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 166312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return best_chan; 166412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 166512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinEXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel); 166612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 166712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 166812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_get_group_entry - get group entry with index idx 166912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @tdesc: is the last allocated slot in the group. 167012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 167112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot * 167212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx) 167312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 167412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter = tdesc->group_head; 167512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i = 0; 167612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 167712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) { 167812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n", 167912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt); 168012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 168112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 168212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 168312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &tdesc->group_list, chain_node) { 168412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i++ == entry_idx) 168512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 168612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 168712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return iter; 168812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 168912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 169012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 169112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_free_slots - flags descriptor slots for reuse 169212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @slot: Slot to free 169312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Caller must hold &ppc440spe_chan->lock while calling this function 169412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 169512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot, 169612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 169712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 169812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int stride = slot->slots_per_op; 169912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 170012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (stride--) { 170112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->slots_per_op = 0; 170212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot = list_entry(slot->slot_node.next, 170312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 170412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node); 170512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 170612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 170712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 170812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_unmap(struct ppc440spe_adma_chan *chan, 170912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc) 171012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 171112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 src_cnt, dst_cnt; 171212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr; 171312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 171412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 171512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * get the number of sources & destination 171612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * included in this descriptor and unmap 171712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * them all 171812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 171912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt = ppc440spe_desc_get_src_num(desc, chan); 172012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst_cnt = ppc440spe_desc_get_dst_num(desc, chan); 172112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 172212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* unmap destinations */ 172312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(desc->async_tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) { 172412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (dst_cnt--) { 172512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr = ppc440spe_desc_get_dest_addr( 172612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, chan, dst_cnt); 172712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(chan->device->dev, 172812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr, desc->unmap_len, 172912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_FROM_DEVICE); 173012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 173112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 173212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 173312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* unmap sources */ 173412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(desc->async_tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) { 173512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (src_cnt--) { 173612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr = ppc440spe_desc_get_src_addr( 173712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, chan, src_cnt); 173812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(chan->device->dev, 173912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr, desc->unmap_len, 174012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_TO_DEVICE); 174112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 174212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 174312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 174412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 174512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 174612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_run_tx_complete_actions - call functions to be called 174712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * upon completion 174812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 174912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic dma_cookie_t ppc440spe_adma_run_tx_complete_actions( 175012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 175112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 175212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t cookie) 175312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 175412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 175512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 175612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(desc->async_tx.cookie < 0); 175712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (desc->async_tx.cookie > 0) { 175812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = desc->async_tx.cookie; 175912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->async_tx.cookie = 0; 176012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 176112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* call the callback (must not sleep or submit new 176212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * operations to this channel) 176312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 176412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (desc->async_tx.callback) 176512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->async_tx.callback( 176612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->async_tx.callback_param); 176712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 176812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* unmap dma addresses 176912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (unmap_single vs unmap_page?) 177012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 177112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * actually, ppc's dma_unmap_page() functions are empty, so 177212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the following code is just for the sake of completeness 177312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 177412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan && chan->needs_unmap && desc->group_head && 177512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->unmap_len) { 177612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *unmap = 177712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->group_head; 177812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* assume 1 slot per op always */ 177912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 slot_count = unmap->slot_cnt; 178012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 178112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Run through the group list and unmap addresses */ 178212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < slot_count; i++) { 178312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!unmap); 178412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_unmap(chan, unmap); 178512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unmap = unmap->hw_next; 178612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 178712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 178812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 178912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 179012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* run dependent operations */ 179112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_run_dependencies(&desc->async_tx); 179212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 179312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return cookie; 179412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 179512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 179612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 179712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set) 179812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 179912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc, 180012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 180112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 180212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* the client is allowed to attach dependent operations 180312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * until 'ack' is set 180412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 180512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!async_tx_test_ack(&desc->async_tx)) 180612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 180712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 180812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* leave the last descriptor in the chain 180912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * so we can append to it 181012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 181112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (list_is_last(&desc->chain_node, &chan->chain) || 181212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->phys == ppc440spe_chan_get_current_descriptor(chan)) 181312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 181412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 181512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan->device->id != PPC440SPE_XOR_ID) { 181612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* our DMA interrupt handler clears opc field of 181712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * each processed descriptor. For all types of 181812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * operations except for ZeroSum we do not actually 181912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * need ack from the interrupt handler. ZeroSum is a 182012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * special case since the result of this operation 182112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * is available from the handler only, so if we see 182212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * such type of descriptor (which is unprocessed yet) 182312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * then leave it in chain. 182412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 182512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *cdb = desc->hw_desc; 182612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cdb->opc == DMA_CDB_OPC_DCHECK128) 182712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 182812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 182912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 183012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n", 183112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->phys, desc->idx, desc->slots_per_op); 183212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 183312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&desc->chain_node); 183412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_free_slots(desc, chan); 183512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 183612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 183712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 183812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 183912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine 184012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * which runs through the channel CDBs list until reach the descriptor 184112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * currently processed. When routine determines that all CDBs of group 184212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * are completed then corresponding callbacks (if any) are called and slots 184312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * are freed. 184412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 184512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan) 184612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 184712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL; 184812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t cookie = 0; 184912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 current_desc = ppc440spe_chan_get_current_descriptor(chan); 185012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int busy = ppc440spe_chan_is_busy(chan); 185112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int seen_current = 0, slot_cnt = 0, slots_per_op = 0; 185212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 185312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n", 185412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id, __func__); 185512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 185612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!current_desc) { 185712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* There were no transactions yet, so 185812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * nothing to clean 185912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 186012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return; 186112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 186212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 186312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* free completed slots from the chain starting with 186412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the oldest descriptor 186512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 186612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(iter, _iter, &chan->chain, 186712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 186812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d " 186912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "busy: %d this_desc: %#llx next_desc: %#x " 187012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "cur: %#x ack: %d\n", 187112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->async_tx.cookie, iter->idx, busy, iter->phys, 187212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_get_link(iter, chan), current_desc, 187312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin async_tx_test_ack(&iter->async_tx)); 187412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prefetch(_iter); 187512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prefetch(&_iter->async_tx); 187612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 187712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do not advance past the current descriptor loaded into the 187812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * hardware channel,subsequent descriptors are either in process 187912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * or have not been submitted 188012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 188112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (seen_current) 188212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 188312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 188412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* stop the search if we reach the current descriptor and the 188512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * channel is busy, or if it appears that the current descriptor 188612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * needs to be re-read (i.e. has been appended to) 188712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 188812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->phys == current_desc) { 188912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(seen_current++); 189012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (busy || ppc440spe_desc_get_link(iter, chan)) { 189112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* not all descriptors of the group have 189212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * been completed; exit. 189312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 189412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 189512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 189612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 189712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 189812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* detect the start of a group transaction */ 189912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!slot_cnt && !slots_per_op) { 190012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = iter->slot_cnt; 190112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = iter->slots_per_op; 190212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slot_cnt <= slots_per_op) { 190312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = 0; 190412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = 0; 190512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 190612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 190712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 190812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slot_cnt) { 190912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!group_start) 191012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = iter; 191112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt -= slots_per_op; 191212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 191312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 191412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* all the members of a group are complete */ 191512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slots_per_op != 0 && slot_cnt == 0) { 191612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter; 191712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int end_of_chain = 0; 191812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 191912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* clean up the group */ 192012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = group_start->slot_cnt; 192112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin grp_iter = group_start; 192212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe_from(grp_iter, _grp_iter, 192312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &chan->chain, chain_node) { 192412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 192512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = ppc440spe_adma_run_tx_complete_actions( 192612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin grp_iter, chan, cookie); 192712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 192812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt -= slots_per_op; 192912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin end_of_chain = ppc440spe_adma_clean_slot( 193012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin grp_iter, chan); 193112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (end_of_chain && slot_cnt) { 193212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Should wait for ZeroSum completion */ 193312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cookie > 0) 19344d4e58de32a192fea65ab84509d17d199bd291c8Russell King - ARM Linux chan->common.completed_cookie = cookie; 193512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return; 193612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 193712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 193812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slot_cnt == 0 || end_of_chain) 193912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 194012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 194112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 194212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* the group should be complete at this point */ 194312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(slot_cnt); 194412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 194512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = 0; 194612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = NULL; 194712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (end_of_chain) 194812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 194912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 195012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 195112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (slots_per_op) /* wait for group completion */ 195212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 195312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 195412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan, 195512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie); 195612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 195712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_adma_clean_slot(iter, chan)) 195812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 195912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 196012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 196112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!seen_current); 196212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 196312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cookie > 0) { 19644d4e58de32a192fea65ab84509d17d199bd291c8Russell King - ARM Linux chan->common.completed_cookie = cookie; 196512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\tcompleted cookie %d\n", cookie); 196612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 196712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 196812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 196912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 197012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 197112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_tasklet - clean up watch-dog initiator 197212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 197312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_tasklet(unsigned long data) 197412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 197512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan = (struct ppc440spe_adma_chan *) data; 197612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 197712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING); 197812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __ppc440spe_adma_slot_cleanup(chan); 197912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock(&chan->lock); 198012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 198112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 198212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 198312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_slot_cleanup - clean up scheduled initiator 198412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 198512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan) 198612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 198712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&chan->lock); 198812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __ppc440spe_adma_slot_cleanup(chan); 198912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 199012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 199112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 199212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 199312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_alloc_slots - allocate free slots (if any) 199412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 199512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots( 199612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, int num_slots, 199712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slots_per_op) 199812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 199912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter = NULL, *_iter; 200012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *alloc_start = NULL; 200112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct list_head chain = LIST_HEAD_INIT(chain); 200212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slots_found, retry = 0; 200312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 200412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 200512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!num_slots || !slots_per_op); 200612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* start search from the last allocated descrtiptor 200712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * if a contiguous allocation can not be found start searching 200812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * from the beginning of the list 200912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 201012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinretry: 201112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_found = 0; 201212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (retry == 0) 201312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = chan->last_used; 201412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 201512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(&chan->all_slots, 201612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 201712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node); 201812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots, 201912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node) { 202012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prefetch(_iter); 202112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prefetch(&_iter->async_tx); 202212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->slots_per_op) { 202312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_found = 0; 202412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 202512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 202612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 202712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* start the allocation if the slot is correctly aligned */ 202812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!slots_found++) 202912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin alloc_start = iter; 203012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 203112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slots_found == num_slots) { 203212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *alloc_tail = NULL; 203312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *last_used = NULL; 203412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 203512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = alloc_start; 203612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (num_slots) { 203712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 203812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* pre-ack all but the last descriptor */ 203912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (num_slots != slots_per_op) 204012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin async_tx_ack(&iter->async_tx); 204112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 204212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_add_tail(&iter->chain_node, &chain); 204312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin alloc_tail = iter; 204412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->async_tx.cookie = 0; 204512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 204612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->flags = 0; 204712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->slot_cnt = num_slots; 204812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->xor_check_result = NULL; 204912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < slots_per_op; i++) { 205012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->slots_per_op = slots_per_op - i; 205112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin last_used = iter; 205212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->slot_node.next, 205312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 205412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node); 205512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 205612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin num_slots -= slots_per_op; 205712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 205812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin alloc_tail->group_head = alloc_start; 205912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin alloc_tail->async_tx.cookie = -EBUSY; 206012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_splice(&chain, &alloc_tail->group_list); 206112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->last_used = last_used; 206212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return alloc_tail; 206312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 206412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 206512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!retry++) 206612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto retry; 206712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 206812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* try to free some slots if the allocation fails */ 206912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_schedule(&chan->irq_tasklet); 207012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 207112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 207212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 207312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 207412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_alloc_chan_resources - allocate pools for CDB slots 207512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 207612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan) 207712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 207812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 207912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *slot = NULL; 208012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin char *hw_desc; 208112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i, db_sz; 208212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int init; 208312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 208412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 208512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin init = ppc440spe_chan->slots_allocated ? 0 : 1; 208612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->chan_id = ppc440spe_chan->device->id; 208712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 208812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Allocate descriptor slots */ 208912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i = ppc440spe_chan->slots_allocated; 209012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID) 209112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin db_sz = sizeof(struct dma_cdb); 209212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 209312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin db_sz = sizeof(struct xor_cb); 209412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 209512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) { 209612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot), 209712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin GFP_KERNEL); 209812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!slot) { 209912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_INFO "SPE ADMA Channel only initialized" 210012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " %d descriptor slots", i--); 210112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 210212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 210312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 210412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt; 210512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->hw_desc = (void *) &hw_desc[i * db_sz]; 210612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_async_tx_descriptor_init(&slot->async_tx, chan); 210712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->async_tx.tx_submit = ppc440spe_adma_tx_submit; 210812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&slot->chain_node); 210912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&slot->slot_node); 211012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&slot->group_list); 211112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz; 211212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->idx = i; 211312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 211412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 211512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->slots_allocated++; 211612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots); 211712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 211812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 211912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 212012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i && !ppc440spe_chan->last_used) { 212112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->last_used = 212212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_entry(ppc440spe_chan->all_slots.next, 212312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 212412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node); 212512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 212612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 212712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 212812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: allocated %d descriptor slots\n", 212912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, i); 213012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 213112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* initialize the channel and the chain with a null operation */ 213212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (init) { 213312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (ppc440spe_chan->device->id) { 213412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 213512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 213612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->hw_chain_inited = 0; 213712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Use WXOR for self-testing */ 213812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!ppc440spe_r6_tchan) 213912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_r6_tchan = ppc440spe_chan; 214012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 214112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 214212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_start_null_xor(ppc440spe_chan); 214312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 214412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 214512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 214612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 214712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->needs_unmap = 1; 214812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 214912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 215012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return (i > 0) ? i : -ENOMEM; 215112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 215212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 215312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 215412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_assign_cookie - assign a cookie 215512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 215612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic dma_cookie_t ppc440spe_desc_assign_cookie( 215712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 215812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc) 215912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 216012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t cookie = chan->common.cookie; 216112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 216212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie++; 216312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cookie < 0) 216412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = 1; 216512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->common.cookie = desc->async_tx.cookie = cookie; 216612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return cookie; 216712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 216812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 216912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 217012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_rxor_set_region_data - 217112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 217212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc, 217312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 xor_arg_no, u32 mask) 217412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 217512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 217612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 217712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->ops[xor_arg_no].h |= mask; 217812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 217912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 218012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 218112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_rxor_set_src - 218212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 218312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc, 218412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 xor_arg_no, dma_addr_t addr) 218512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 218612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 218712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 218812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE; 218912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->ops[xor_arg_no].l = addr; 219012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 219112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 219212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 219312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_rxor_set_mult - 219412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 219512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc, 219612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 xor_arg_no, u8 idx, u8 mult) 219712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 219812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 219912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 220012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8); 220112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 220212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 220312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 220412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold 220512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * has been achieved 220612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 220712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan) 220812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 220912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n", 221012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id, chan->pending); 221112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 221212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) { 221312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->pending = 0; 221412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_append(chan); 221512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 221612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 221712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 221812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 221912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_tx_submit - submit new descriptor group to the channel 222012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (it's not necessary that descriptors will be submitted to the h/w 222112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * chains too right now) 222212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 222312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx) 222412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 222512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc; 222612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan); 222712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail; 222812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 222912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slots_per_op; 223012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t cookie; 223112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 223212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = tx_to_ppc440spe_adma_slot(tx); 223312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 223412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 223512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = group_start->slot_cnt; 223612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = group_start->slots_per_op; 223712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 223812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&chan->lock); 223912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 224012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = ppc440spe_desc_assign_cookie(chan, sw_desc); 224112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 224212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(list_empty(&chan->chain))) { 224312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* first peer */ 224412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_splice_init(&sw_desc->group_list, &chan->chain); 224512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan_first_cdb[chan->device->id] = group_start; 224612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 224712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* isn't first peer, bind CDBs to chain */ 224812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin old_chain_tail = list_entry(chan->chain.prev, 224912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 225012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 225112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_splice_init(&sw_desc->group_list, 225212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &old_chain_tail->chain_node); 225312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* fix up the hardware chain */ 225412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_link(chan, old_chain_tail, group_start); 225512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 225612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 225712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* increment the pending count by the number of operations */ 225812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->pending += slot_cnt / slots_per_op; 225912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_check_threshold(chan); 226012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 226112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 226212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, 226312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n", 226412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id, __func__, 226512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.cookie, sw_desc->idx, sw_desc); 226612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 226712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return cookie; 226812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 226912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 227012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 227112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation 227212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 227312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt( 227412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, unsigned long flags) 227512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 227612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 227712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 227812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op; 227912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 228012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 228112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 228212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 228312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id, 228412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 228512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 228612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 228712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = slots_per_op = 1; 228812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 228912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op); 229012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 229112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 229212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan); 229312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start->unmap_len = 0; 229412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 229512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 229612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 229712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 229812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 229912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 230012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 230112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 230212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation 230312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 230412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy( 230512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t dma_dest, 230612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t dma_src, size_t len, unsigned long flags) 230712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 230812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 230912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 231012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op; 231112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 231212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 231312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 231412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!len)) 231512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 231612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 2317427cdf19b97e509e21e5d347e18d8b0b34723dfcColy Li BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT); 231812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 231912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 232012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 232112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 232212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s len: %u int_en %d\n", 232312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, __func__, len, 232412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags & DMA_PREP_INTERRUPT ? 1 : 0); 232512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = slots_per_op = 1; 232612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 232712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op); 232812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 232912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 233012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_memcpy(group_start, flags); 233112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_set_dest(group_start, dma_dest, 0); 233212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0); 233312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len); 233412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->unmap_len = len; 233512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 233612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 233712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 233812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 233912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 234012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 234112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 234212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 234312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_memset - prepare CDB for a MEMSET operation 234412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 234512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memset( 234612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t dma_dest, int value, 234712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size_t len, unsigned long flags) 234812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 234912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 235012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 235112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op; 235212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 235312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 235412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 235512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!len)) 235612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 235712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 2358427cdf19b97e509e21e5d347e18d8b0b34723dfcColy Li BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT); 235912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 236012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 236112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 236212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 236312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s cal: %u len: %u int_en %d\n", 236412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, __func__, value, len, 236512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags & DMA_PREP_INTERRUPT ? 1 : 0); 236612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 236712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = slots_per_op = 1; 236812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 236912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op); 237012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 237112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 237212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_memset(group_start, value, flags); 237312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_set_dest(group_start, dma_dest, 0); 237412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len); 237512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->unmap_len = len; 237612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 237712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 237812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 237912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 238012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 238112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 238212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 238312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 238412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation 238512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 238612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor( 238712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t dma_dest, 238812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dma_src, u32 src_cnt, size_t len, 238912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long flags) 239012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 239112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 239212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 239312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op; 239412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 239512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 239612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 239712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id, 239812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_dest, dma_src, src_cnt)); 239912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!len)) 240012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 2401427cdf19b97e509e21e5d347e18d8b0b34723dfcColy Li BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 240212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 240312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 240412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n", 240512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, __func__, src_cnt, len, 240612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags & DMA_PREP_INTERRUPT ? 1 : 0); 240712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 240812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 240912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op); 241012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 241112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op); 241212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 241312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 241412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_xor(group_start, src_cnt, flags); 241512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_set_dest(group_start, dma_dest, 0); 241612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (src_cnt--) 241712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_memcpy_xor_set_src(group_start, 241812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_src[src_cnt], src_cnt); 241912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len); 242012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->unmap_len = len; 242112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 242212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 242312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 242412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 242512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 242612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 242712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 242812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline void 242912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc, 243012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt); 243112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor); 243212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 243312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 243412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_init_dma2rxor_slot - 243512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 243612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_init_dma2rxor_slot( 243712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 243812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *src, int src_cnt) 243912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 244012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 244112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 244212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* initialize CDB */ 244312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) { 244412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i, 244512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt, (u32)src[i]); 244612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 244712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 244812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 244912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 245012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma01_prep_mult - 245112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for Q operation where destination is also the source 245212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 245312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult( 245412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan, 245512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 245612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf, size_t len, unsigned long flags) 245712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 245812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL; 245912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 246012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 246112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 246212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_WXOR, &op); 246312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = 2; 246412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 246512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 246612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 246712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* use WXOR, each descriptor occupies one slot */ 246812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 246912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 247012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 247112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 247212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 247312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 247412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 247512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bits(op, &sw_desc->flags); 247612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->src_cnt = src_cnt; 247712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt = dst_cnt; 247812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* First descriptor, zero data in the destination and copy it 247912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * to q page using MULTICAST transfer. 248012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 248112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&sw_desc->group_list, 248212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 248312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 248412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 248512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 248612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 248712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 248812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 248912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 249012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 249112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MULTICAST; 249212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 249312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 249412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, dst[0], 0); 249512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1); 249612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 249712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src[0]); 249812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 249912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 250012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 250112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 250212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Second descriptor, multiply data from the q page 250312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * and store the result in real destination. 250412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 250512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 250612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 250712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 250812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 250912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 251012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 251112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 251212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 251312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 251412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 251512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 251612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 251712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 251812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB, dst[1]); 251912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 252012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, dst[0], 0); 252112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 252212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 252312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1, scf[0]); 252412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 252512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 252612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 252712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 252812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 252912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 253012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 253112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc; 253212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 253312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 253412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 253512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma01_prep_sum_product - 253612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also 253712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the source. 253812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 253912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product( 254012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan, 254112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dst, dma_addr_t *src, int src_cnt, 254212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf, size_t len, unsigned long flags) 254312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 254412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL; 254512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 254612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 254712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 254812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_WXOR, &op); 254912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = 3; 255012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 255112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 255212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 255312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR, each descriptor occupies one slot */ 255412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 255512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 255612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 255712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 255812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 255912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 256012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 256112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bits(op, &sw_desc->flags); 256212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->src_cnt = src_cnt; 256312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt = 1; 256412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 1st descriptor, src[1] data to q page and zero destination */ 256512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&sw_desc->group_list, 256612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 256712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 256812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 256912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 257012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 257112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 257212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 257312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 257412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MULTICAST; 257512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 257612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 257712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *dst, 0); 257812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 0, 257912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->qdest, 1); 258012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 258112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src[1]); 258212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 258312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 258412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 258512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 2nd descriptor, multiply src[1] data and store the 258612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * result in destination */ 258712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 258812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 258912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 259012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 259112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 259212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 259312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 259412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 259512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 259612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 259712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 259812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 259912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 260012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 260112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 260212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 260312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->qdest); 260412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 260512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *dst, 0); 260612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 260712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1, scf[1]); 260812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 260912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 261012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 261112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 261212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 3rd descriptor, multiply src[0] data and xor it 261312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * with destination 261412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 261512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 261612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 261712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 261812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 261912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 262012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 262112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 262212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 262312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 262412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 262512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 262612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 262712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 262812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src[0]); 262912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 263012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *dst, 0); 263112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 263212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1, scf[0]); 263312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 263412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 263512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 263612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 263712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 263812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 263912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 264012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc; 264112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 264212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 264312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq( 264412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan, 264512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 264612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf, size_t len, unsigned long flags) 264712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 264812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 264912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter; 265012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 265112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned char mult = 1; 265212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 265312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n", 265412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dst_cnt, src_cnt, len); 265512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* select operations WXOR/RXOR depending on the 265612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * source addresses of operators and the number 265712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * of destinations (RXOR support only Q-parity calculations) 265812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 265912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_WXOR, &op); 266012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) { 266112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* no active RXOR; 266212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * do RXOR if: 266312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - there are more than 1 source, 266412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - len is aligned on 512-byte boundary, 266512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - source addresses fit to one of 4 possible regions. 266612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 266712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt > 1 && 266812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin !(len & MQ0_CF2H_RXOR_BS_MASK) && 266912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (src[0] + len) == src[1]) { 267012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* may do RXOR R1 R2 */ 267112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR, &op); 267212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt != 2) { 267312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* may try to enhance region of RXOR */ 267412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((src[1] + len) == src[2]) { 267512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 R3 */ 267612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR123, 267712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &op); 267812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if ((src[1] + len * 2) == src[2]) { 267912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 R4 */ 268012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR124, &op); 268112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if ((src[1] + len * 3) == src[2]) { 268212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 R5 */ 268312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR125, 268412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &op); 268512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 268612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 */ 268712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR12, 268812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &op); 268912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 269012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 269112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 */ 269212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR12, &op); 269312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 269412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 269512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 269612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_RXOR, &op)) { 269712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* can not do this operation with RXOR */ 269812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_RXOR_RUN, 269912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &ppc440spe_rxor_state); 270012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 270112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* can do; set block size right now */ 270212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_rxor_block_size(len); 270312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 270412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 270512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 270612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Number of necessary slots depends on operation type selected */ 270712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_RXOR, &op)) { 270812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is a WXOR only chain. Need descriptors for each 270912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * source to GF-XOR them with WXOR, and need descriptors 271012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for each destination to zero them with WXOR 271112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 271212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = src_cnt; 271312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 271412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_P) { 271512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt++; 271612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_P, &op); 271712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 271812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_Q) { 271912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt++; 272012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_Q, &op); 272112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 272212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 272312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Need 1/2 descriptor for RXOR operation, and 272412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * need (src_cnt - (2 or 3)) for WXOR of sources 272512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * remained (if any) 272612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 272712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = dst_cnt; 272812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 272912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_P) 273012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_P, &op); 273112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_Q) 273212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_Q, &op); 273312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 273412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_RXOR12, &op)) 273512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt += src_cnt - 2; 273612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 273712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt += src_cnt - 3; 273812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 273912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Thus we have either RXOR only chain or 274012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * mixed RXOR/WXOR 274112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 274212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slot_cnt == dst_cnt) 274312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR only chain */ 274412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_WXOR, &op); 274512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 274612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 274712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 274812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* for both RXOR/WXOR each descriptor occupies one slot */ 274912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 275012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 275112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt, 275212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags, op); 275312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 275412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* setup dst/src/mult */ 275512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n", 275612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dst[0], dst[1]); 275712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_dest(sw_desc, dst, flags); 275812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (src_cnt--) { 275912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt], 276012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt); 276112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 276212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* NOTE: "Multi = 0 is equivalent to = 1" as it 276312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf 276412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * doesn't work for RXOR with DMA0/1! Instead, multi=0 276512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * leads to zeroing source data after RXOR. 276612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * So, for P case set-up mult=1 explicitly. 276712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 276812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 276912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult = scf[src_cnt]; 277012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src_mult(sw_desc, 277112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult, src_cnt, dst_cnt - 1); 277212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 277312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 277412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup byte count foreach slot just allocated */ 277512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 277612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, 277712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 277812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, 277912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan, len); 278012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 278112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 278212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 278312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 278412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 278512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc; 278612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 278712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 278812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq( 278912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan, 279012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 279112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf, size_t len, unsigned long flags) 279212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 279312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, descs_per_op; 279412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter; 279512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 279612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned char mult = 1; 279712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 279812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!dst_cnt); 279912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n", 280012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dst_cnt, src_cnt, len);*/ 280112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 280212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 280312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len); 280412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (descs_per_op < 0) { 280512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 280612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 280712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 280812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 280912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* depending on number of sources we have 1 or 2 RXOR chains */ 281012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = descs_per_op * dst_cnt; 281112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 281212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 281312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 281412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op = slot_cnt; 281512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 281612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 281712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt, 281812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin --op ? 0 : flags); 281912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 282012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len); 282112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 282212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 282312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_init_rxor_cursor(&(iter->rxor_cursor)); 282412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->rxor_cursor.len = len; 282512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->descs_per_op = descs_per_op; 282612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 282712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op = 0; 282812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 282912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op++; 283012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (op % descs_per_op == 0) 283112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_init_dma2rxor_slot(iter, src, 283212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt); 283312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(!list_is_last(&iter->chain_node, 283412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list))) { 283512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 283612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = 283712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_entry(iter->chain_node.next, 283812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 283912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 284012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_xor_set_link(iter, iter->hw_next); 284112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 284212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is the last descriptor. */ 284312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 284412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 284512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 284612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 284712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* fixup head descriptor */ 284812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt = dst_cnt; 284912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_P) 285012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_P, &sw_desc->flags); 285112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_Q) 285212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags); 285312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 285412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* setup dst/src/mult */ 285512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_dest(sw_desc, dst, flags); 285612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 285712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (src_cnt--) { 285812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* handle descriptors (if dst_cnt == 2) inside 285912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the ppc440spe_adma_pq_set_srcxxx() functions 286012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 286112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt], 286212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt); 286312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 286412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult = scf[src_cnt]; 286512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src_mult(sw_desc, 286612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult, src_cnt, dst_cnt - 1); 286712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 286812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 286912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 287012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_rxor_block_size(len); 287112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc; 287212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 287312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 287412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 287512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation 287612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 287712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq( 287812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, 287912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt, const unsigned char *scf, 288012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size_t len, unsigned long flags) 288112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 288212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 288312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL; 288412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int dst_cnt = 0; 288512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 288612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 288712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 288812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id, 288912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst, src, src_cnt)); 289012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!len); 2891427cdf19b97e509e21e5d347e18d8b0b34723dfcColy Li BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 289212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!src_cnt); 289312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 289412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt == 1 && dst[1] == src[0]) { 289512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t dest[2]; 289612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 289712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* dst[1] is real destination (Q) */ 289812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dest[0] = dst[1]; 289912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is the page to multicast source data to */ 290012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dest[1] = ppc440spe_chan->qdest; 290112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan, 290212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dest, 2, src, src_cnt, scf, len, flags); 290312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 290412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 290512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 290612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt == 2 && dst[1] == src[1]) { 290712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan, 290812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &dst[1], src, 2, scf, len, flags); 290912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 291012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 291112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 291212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(flags & DMA_PREP_PQ_DISABLE_P)) { 291312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!dst[0]); 291412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst_cnt++; 291512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags |= DMA_PREP_ZERO_P; 291612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 291712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 291812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(flags & DMA_PREP_PQ_DISABLE_Q)) { 291912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!dst[1]); 292012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst_cnt++; 292112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags |= DMA_PREP_ZERO_Q; 292212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 292312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 292412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!dst_cnt); 292512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 292612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 292712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n", 292812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, __func__, src_cnt, len, 292912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags & DMA_PREP_INTERRUPT ? 1 : 0); 293012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 293112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (ppc440spe_chan->device->id) { 293212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 293312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 293412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan, 293512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst, dst_cnt, src, src_cnt, scf, 293612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len, flags); 293712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 293812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 293912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 294012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan, 294112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst, dst_cnt, src, src_cnt, scf, 294212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len, flags); 294312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 294412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 294512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 294612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 294712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 294812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 294912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 295012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for 295112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * a PQ_ZERO_SUM operation 295212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 295312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum( 295412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, 295512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt, const unsigned char *scf, size_t len, 295612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin enum sum_check_flags *pqres, unsigned long flags) 295712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 295812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 295912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *iter; 296012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t pdest, qdest; 296112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op, idst, dst_cnt; 296212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 296312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 296412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 296512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_PQ_DISABLE_P) 296612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pdest = 0; 296712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 296812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pdest = pq[0]; 296912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 297012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_PQ_DISABLE_Q) 297112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qdest = 0; 297212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 297312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qdest = pq[1]; 297412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 297512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id, 297612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src, src_cnt, scf)); 297712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 297812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Always use WXOR for P/Q calculations (two destinations). 297912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Need 1 or 2 extra slots to verify results are zero. 298012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 298112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idst = dst_cnt = (pdest && qdest) ? 2 : 1; 298212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 298312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* One additional slot per destination to clone P/Q 298412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * before calculation (we have to preserve destinations). 298512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 298612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = src_cnt + dst_cnt * 2; 298712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = 1; 298812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 298912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 299012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 299112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op); 299212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 299312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt); 299412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 299512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup byte count for each slot just allocated */ 299612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 299712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 299812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 299912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len); 300012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 300112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 300212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 300312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (pdest) { 300412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 300512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 300612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 300712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = sw_desc->group_head; 300812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 300912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 301012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 301112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 301212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 301312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 301412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 301512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->src_cnt = 0; 301612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->dst_cnt = 0; 301712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 0, 301812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->pdest, 0); 301912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest); 302012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 302112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len); 302212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = 0; 302312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* override pdest to preserve original P */ 302412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pdest = ppc440spe_chan->pdest; 302512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 302612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (qdest) { 302712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 302812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 302912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 303012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&sw_desc->group_list, 303112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 303212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 303312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 303412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 303512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (pdest) { 303612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 303712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 303812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 303912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 304012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 304112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 304212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 304312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 304412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 304512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 304612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 304712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->src_cnt = 0; 304812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->dst_cnt = 0; 304912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 0, 305012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->qdest, 0); 305112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest); 305212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 305312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len); 305412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = 0; 305512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* override qdest to preserve original Q */ 305612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qdest = ppc440spe_chan->qdest; 305712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 305812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 305912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup destinations for P/Q ops */ 306012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest); 306112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 306212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup zero QWORDs into DCHECK CDBs */ 306312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idst = dst_cnt; 306412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_reverse(iter, &sw_desc->group_list, 306512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 306612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 306712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * The last CDB corresponds to Q-parity check, 306812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the one before last CDB corresponds 306912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * P-parity check 307012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 307112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (idst == DMA_DEST_MAX_NUM) { 307212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (idst == dst_cnt) { 307312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_QCHECK, 307412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags); 307512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 307612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_PCHECK, 307712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags); 307812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 307912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 308012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (qdest) { 308112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_QCHECK, 308212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags); 308312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 308412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_PCHECK, 308512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags); 308612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 308712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 308812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->xor_check_result = pqres; 308912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 309012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 309112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * set it to zero, if check fail then result will 309212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * be updated 309312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 309412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *iter->xor_check_result = 0; 309512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dcheck(iter, ppc440spe_chan, 309612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_qword); 309712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 309812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(--dst_cnt)) 309912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 310012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 310112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 310212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup sources and mults for P/Q ops */ 310312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_continue_reverse(iter, &sw_desc->group_list, 310412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 310512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 310612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 mult_dst; 310712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 310812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 310912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 311012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB, 311112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src[src_cnt - 1]); 311212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (qdest) { 311312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 : 311412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1; 311512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, 311612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_MULT1_OFF, 311712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst, 311812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin scf[src_cnt - 1]); 311912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 312012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(--src_cnt)) 312112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 312212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 312312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 312412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 312512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 312612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 312712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 312812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 312912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for 313012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * XOR ZERO_SUM operation 313112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 313212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum( 313312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt, 313412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size_t len, enum sum_check_flags *result, unsigned long flags) 313512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 313612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_async_tx_descriptor *tx; 313712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t pq[2]; 313812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 313912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* validate P, disable Q */ 314012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pq[0] = src[0]; 314112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pq[1] = 0; 314212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags |= DMA_PREP_PQ_DISABLE_Q; 314312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 314412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1], 314512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt - 1, 0, len, 314612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin result, flags); 314712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return tx; 314812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 314912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 315012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 315112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_set_dest - set destination address into descriptor 315212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 315312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc, 315412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index) 315512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 315612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 315712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 315812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(index >= sw_desc->dst_cnt); 315912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 316012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 316112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 316212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 316312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 316412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 316512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* to do: support transfers lengths > 316612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT 316712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 316812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(sw_desc->group_head, 316912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan, 0, addr, index); 317012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 317112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 317212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_get_group_entry(sw_desc, index); 317312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(sw_desc, 317412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan, 0, addr, index); 317512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 317612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 317712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 317812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 317912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter, 318012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, dma_addr_t addr) 318112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 318212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* To clear destinations update the descriptor 318312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (P or Q depending on index) as follows: 318412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * addr is destination (0 corresponds to SG2): 318512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 318612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0); 318712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 318812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* ... and the addr is source: */ 318912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr); 319012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 319112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* addr is always SG2 then the mult is always DST1 */ 319212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 319312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1, 1); 319412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 319512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 319612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 319712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_pq_set_dest - set destination address into descriptor 319812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for the PQXOR operation 319912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 320012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc, 320112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *addrs, unsigned long flags) 320212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 320312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 320412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 320512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t paddr, qaddr; 320612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr = 0, ppath, qpath; 320712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index = 0, i; 320812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 320912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 321012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 321112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_PQ_DISABLE_P) 321212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr = 0; 321312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 321412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr = addrs[0]; 321512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 321612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_PQ_DISABLE_Q) 321712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qaddr = 0; 321812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 321912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qaddr = addrs[1]; 322012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 322112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!paddr || !qaddr) 322212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr = paddr ? paddr : qaddr; 322312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 322412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 322512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 322612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 322712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* walk through the WXOR source list and set P/Q-destinations 322812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for each slot: 322912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 323012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 323112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is WXOR-only chain; may have 1/2 zero descs */ 323212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 323312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index++; 323412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 323512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index++; 323612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 323712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, index); 323812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (addr) { 323912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* one destination */ 324012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, 324112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list, chain_node) 324212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 324312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, addr, 0); 324412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 324512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* two destinations */ 324612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, 324712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list, chain_node) { 324812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 324912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, paddr, 0); 325012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 325112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, qaddr, 1); 325212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 325312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 325412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 325512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index) { 325612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* To clear destinations update the descriptor 325712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (1st,2nd, or both depending on flags) 325812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 325912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index = 0; 326012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, 326112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) { 326212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry( 326312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc, index++); 326412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_zero_op(iter, chan, 326512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr); 326612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 326712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 326812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, 326912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) { 327012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry( 327112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc, index++); 327212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_zero_op(iter, chan, 327312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qaddr); 327412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 327512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 327612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return; 327712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 327812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 327912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is RXOR-only or RXOR/WXOR mixed chain */ 328012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 328112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* If we want to include destination into calculations, 328212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * then make dest addresses cued with mult=1 (XOR). 328312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 328412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ? 328512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB : 328612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE | 328712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << DMA_CUED_MULT1_OFF); 328812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ? 328912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB : 329012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE | 329112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << DMA_CUED_MULT1_OFF); 329212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 329312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup destination(s) in RXOR slot(s) */ 329412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, index++); 329512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 329612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr ? ppath : qpath, 329712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr ? paddr : qaddr, 0); 329812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addr) { 329912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* two destinations */ 330012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 330112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index++); 330212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 330312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qpath, qaddr, 0); 330412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 330512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 330612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) { 330712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup destination(s) in remaining WXOR 330812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * slots 330912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 331012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 331112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index); 331212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (addr) { 331312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* one destination */ 331412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, 331512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list, 331612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) 331712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr( 331812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter, chan, 331912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, 332012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr, 0); 332112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 332212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 332312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* two destinations */ 332412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, 332512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list, 332612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 332712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr( 332812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter, chan, 332912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, 333012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr, 0); 333112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr( 333212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter, chan, 333312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, 333412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qaddr, 1); 333512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 333612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 333712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 333812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 333912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 334012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 334112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 334212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 334312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA2 descriptors have only 1 destination, so there are 334412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * two chains - one for each dest. 334512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * If we want to include destination into calculations, 334612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * then make dest addresses cued with mult=1 (XOR). 334712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 334812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ? 334912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB : 335012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE | 335112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << DMA_CUED_MULT1_OFF); 335212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 335312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ? 335412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB : 335512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE | 335612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << DMA_CUED_MULT1_OFF); 335712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 335812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 0); 335912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < sw_desc->descs_per_op; i++) { 336012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 336112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr ? ppath : qpath, 336212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr ? paddr : qaddr, 0); 336312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 336412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 336512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 336612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 336712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 336812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addr) { 336912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Two destinations; setup Q here */ 337012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 337112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->descs_per_op); 337212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < sw_desc->descs_per_op; i++) { 337312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, 337412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan, qpath, qaddr, 0); 337512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 337612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 337712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 337812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 337912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 338012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 338112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 338212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 338312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 338412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 338512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 338612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor 338712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for the PQ_ZERO_SUM operation 338812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 338912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pqzero_sum_set_dest( 339012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, 339112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t paddr, dma_addr_t qaddr) 339212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 339312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter, *end; 339412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 339512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr = 0; 339612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int idx; 339712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 339812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 339912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 340012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* walk through the WXOR source list and set P/Q-destinations 340112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for each slot 340212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 340312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idx = (paddr && qaddr) ? 2 : 1; 340412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set end */ 340512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_reverse(end, &sw_desc->group_list, 340612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 340712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(--idx)) 340812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 340912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 341012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set start */ 341112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idx = (paddr && qaddr) ? 2 : 1; 341212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, idx); 341312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 341412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (paddr && qaddr) { 341512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* two destinations */ 341612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &sw_desc->group_list, 341712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 341812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(iter == end)) 341912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 342012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 342112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, paddr, 0); 342212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 342312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, qaddr, 1); 342412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 342512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 342612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* one destination */ 342712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr = paddr ? paddr : qaddr; 342812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &sw_desc->group_list, 342912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 343012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(iter == end)) 343112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 343212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 343312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, addr, 0); 343412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 343512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 343612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 343712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* The remaining descriptors are DATACHECK. These have no need in 343812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * destination. Actually, these destinations are used there 343912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * as sources for check operation. So, set addr as source. 344012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 344112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr); 344212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 344312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addr) { 344412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin end = list_entry(end->chain_node.next, 344512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 344612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr); 344712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 344812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 344912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 345012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 345112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor 345212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 345312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline void ppc440spe_desc_set_xor_src_cnt( 345412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 345512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt) 345612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 345712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *hw_desc = desc->hw_desc; 345812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 345912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc &= ~XOR_CDCR_OAC_MSK; 346012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc |= src_cnt; 346112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 346212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 346312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 346412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_pq_set_src - set source address into descriptor 346512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 346612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc, 346712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index) 346812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 346912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 347012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t haddr = 0; 347112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter = NULL; 347212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 347312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 347412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 347512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 347612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 347712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 347812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain 347912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 348012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 348112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR-only or RXOR/WXOR operation */ 348212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int iskip = test_bit(PPC440SPE_DESC_RXOR12, 348312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags) ? 2 : 3; 348412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 348512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == 0) { 348612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 1st slot (RXOR) */ 348712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* setup sources region (R1-2-3, R1-2-4, 348812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * or R1-2-5) 348912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 349012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_RXOR12, 349112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) 349212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_RXOR12 << 349312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 349412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else if (test_bit(PPC440SPE_DESC_RXOR123, 349512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) 349612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_RXOR123 << 349712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 349812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else if (test_bit(PPC440SPE_DESC_RXOR124, 349912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) 350012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_RXOR124 << 350112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 350212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else if (test_bit(PPC440SPE_DESC_RXOR125, 350312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) 350412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_RXOR125 << 350512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 350612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 350712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 350812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr |= DMA_CUED_XOR_BASE; 350912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 0); 351012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (index < iskip) { 351112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 1st slot (RXOR) 351212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * shall actually set source address only once 351312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * instead of first <iskip> 351412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 351512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = NULL; 351612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 351712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 2nd/3d and next slots (WXOR); 351812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * skip first slot with RXOR 351912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 352012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_CUED_XOR_HB; 352112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 352212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index - iskip + sw_desc->dst_cnt); 352312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 352412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 352512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int znum = 0; 352612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 352712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR-only operation; skip first slots with 352812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * zeroing destinations 352912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 353012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 353112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin znum++; 353212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 353312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin znum++; 353412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 353512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_CUED_XOR_HB; 353612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 353712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index + znum); 353812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 353912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 354012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(iter)) { 354112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr); 354212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 354312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!index && 354412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) && 354512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt == 2) { 354612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* if we have two destinations for RXOR, then 354712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * setup source in the second descr too 354812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 354912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 1); 355012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 355112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr, addr); 355212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 355312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 355412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 355512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 355612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 355712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA2 may do Biskup */ 355812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = sw_desc->group_head; 355912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->dst_cnt == 2) { 356012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* both P & Q calculations required; set P src here */ 356112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_set_src(iter, index, addr); 356212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 356312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is for Q */ 356412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 356512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->descs_per_op); 356612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 356712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_set_src(iter, index, addr); 356812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 356912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 357012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 357112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 357212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 357312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor 357412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 357512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_memcpy_xor_set_src( 357612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, 357712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index) 357812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 357912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 358012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 358112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 358212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = sw_desc->group_head; 358312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 358412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(sw_desc)) 358512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr); 358612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 358712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 358812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 358912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_inc_addr - 359012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 359112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_inc_addr( 359212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 359312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_rxor *cursor, int index, int src_cnt) 359412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 359512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count++; 359612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt - 1) { 359712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count); 359812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cursor->addr_count == XOR_MAX_OPS) { 359912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count); 360012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count = 0; 360112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->desc_count++; 360212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 360312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 360412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 360512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 360612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB 360712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 360812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_dma2rxor_prep_src( 360912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *hdesc, 361012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_rxor *cursor, int index, 361112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, u32 addr) 361212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 361312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int rval = 0; 361412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 sign; 361512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc = hdesc; 361612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 361712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 361812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < cursor->desc_count; i++) { 361912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc = list_entry(hdesc->chain_node.next, 362012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 362112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 362212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 362312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 362412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (cursor->state) { 362512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 0: 362612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (addr == cursor->addrl + cursor->len) { 362712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct RXOR */ 362812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 1; 362912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count++; 363012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 363112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 363212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 363312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR12 << DMA_CUED_REGION_OFF); 363412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 363512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 363612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 363712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cursor->addrl == addr + cursor->len) { 363812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse RXOR */ 363912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 1; 364012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count++; 364112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(cursor->addr_count, &desc->reverse_flags[0]); 364212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 364312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 364412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 364512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR12 << DMA_CUED_REGION_OFF); 364612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 364712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 364812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 364912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 365012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "Cannot build " 365112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "DMA2 RXOR command block.\n"); 365212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 365312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 365412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 365512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 1: 365612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sign = test_bit(cursor->addr_count, 365712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->reverse_flags) 365812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ? -1 : 1; 365912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-2 || (sign == -1 366012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin && addr != cursor->addrl - 2*cursor->len)) { 366112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 0; 366212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 1; 366312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addrl = addr; 366412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 366512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 366612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR12 << DMA_CUED_REGION_OFF); 366712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 366812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 366912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (addr == cursor->addrl + 2*sign*cursor->len) { 367012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 2; 367112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 0; 367212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 367312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 367412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR123 << DMA_CUED_REGION_OFF); 367512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 367612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 367712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 367812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 367912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (addr == cursor->addrl + 3*cursor->len) { 368012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 2; 368112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 0; 368212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 368312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 368412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR124 << DMA_CUED_REGION_OFF); 368512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 368612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 368712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 368812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 368912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (addr == cursor->addrl + 4*cursor->len) { 369012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 2; 369112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 0; 369212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 369312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 369412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR125 << DMA_CUED_REGION_OFF); 369512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 369612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 369712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 369812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 369912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 370012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 0; 370112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 1; 370212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addrl = addr; 370312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 370412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 370512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR12 << DMA_CUED_REGION_OFF); 370612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 370712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 370812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 370912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 371012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 2: 371112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 0; 371212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addrl = addr; 371312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count++; 371412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index) { 371512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 371612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 371712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 371812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 371912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 372012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 372112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return rval; 372212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 372312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 372412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 372512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that 372612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call 372712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 372812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_set_src( 372912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 373012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index, dma_addr_t addr) 373112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 373212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 373312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int k = 0, op = 0, lop = 0; 373412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 373512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* get the RXOR operand which corresponds to index addr */ 373612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (op <= index) { 373712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin lop = op; 373812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (k == XOR_MAX_OPS) { 373912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin k = 0; 374012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc = list_entry(desc->chain_node.next, 374112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 374212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb = desc->hw_desc; 374312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 374412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 374512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) == 374612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (DMA_RXOR12 << DMA_CUED_REGION_OFF)) 374712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op += 2; 374812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 374912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op += 3; 375012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 375112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 375212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(k < 1); 375312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 375412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(k-1, desc->reverse_flags)) { 375512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse operand order; put last op in RXOR group */ 375612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == op - 1) 375712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_src(desc, k - 1, addr); 375812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 375912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct operand order; put first op in RXOR group */ 376012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == lop) 376112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_src(desc, k - 1, addr); 376212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 376312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 376412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 376512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 376612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that 376712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call 376812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 376912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_set_mult( 377012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 377112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index, u8 mult) 377212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 377312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 377412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int k = 0, op = 0, lop = 0; 377512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 377612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* get the RXOR operand which corresponds to index mult */ 377712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (op <= index) { 377812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin lop = op; 377912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (k == XOR_MAX_OPS) { 378012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin k = 0; 378112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc = list_entry(desc->chain_node.next, 378212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 378312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 378412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb = desc->hw_desc; 378512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 378612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 378712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) == 378812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (DMA_RXOR12 << DMA_CUED_REGION_OFF)) 378912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op += 2; 379012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 379112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op += 3; 379212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 379312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 379412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(k < 1); 379512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(k-1, desc->reverse_flags)) { 379612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse order */ 379712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult); 379812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 379912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct order */ 380012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult); 380112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 380212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 380312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 380412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 380512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_init_rxor_cursor - 380612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 380712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor) 380812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 380912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(cursor, 0, sizeof(struct ppc440spe_rxor)); 381012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 2; 381112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 381212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 381312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 381412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into 381512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor for the PQXOR operation 381612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 381712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pq_set_src_mult( 381812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, 381912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned char mult, int index, int dst_pos) 382012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 382112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 382212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 mult_idx, mult_dst; 382312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL; 382412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 382512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 382612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 382712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 382812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 382912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 383012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 383112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int region = test_bit(PPC440SPE_DESC_RXOR12, 383212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags) ? 2 : 3; 383312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 383412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index < region) { 383512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR multipliers */ 383612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 383712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt - 1); 383812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc->dst_cnt == 2) 383912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter1 = ppc440spe_get_group_entry( 384012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc, 0); 384112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 384212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx = DMA_CUED_MULT1_OFF + (index << 3); 384312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst = DMA_CDB_SG_SRC; 384412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 384512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR multiplier */ 384612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 384712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index - region + 384812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt); 384912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx = DMA_CUED_MULT1_OFF; 385012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst = dst_pos ? DMA_CDB_SG_DST2 : 385112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1; 385212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 385312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 385412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int znum = 0; 385512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 385612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR-only; 385712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * skip first slots with destinations (if ZERO_DST has 385812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * place) 385912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 386012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 386112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin znum++; 386212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 386312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin znum++; 386412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 386512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, index + znum); 386612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx = DMA_CUED_MULT1_OFF; 386712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1; 386812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 386912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 387012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(iter)) { 387112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, 387212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx, mult_dst, mult); 387312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 387412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(iter1)) { 387512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* if we have two destinations for RXOR, then 387612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * we've just set Q mult. Set-up P now. 387712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 387812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter1, chan, 387912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx, mult_dst, 1); 388012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 388112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 388212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 388312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 388412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 388512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 388612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = sw_desc->group_head; 388712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc->dst_cnt == 2) { 388812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* both P & Q calculations required; set P mult here */ 388912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_set_mult(iter, index, 1); 389012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 389112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* and then set Q mult */ 389212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 389312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->descs_per_op); 389412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 389512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_set_mult(iter, index, mult); 389612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 389712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 389812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 389912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 390012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 390112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_free_chan_resources - free the resources allocated 390212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 390312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_free_chan_resources(struct dma_chan *chan) 390412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 390512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 390612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter, *_iter; 390712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int in_use_descs = 0; 390812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 390912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 391012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_slot_cleanup(ppc440spe_chan); 391112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 391212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 391312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain, 391412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 391512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin in_use_descs++; 391612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&iter->chain_node); 391712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 391812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe_reverse(iter, _iter, 391912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &ppc440spe_chan->all_slots, slot_node) { 392012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&iter->slot_node); 392112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(iter); 392212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->slots_allocated--; 392312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 392412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->last_used = NULL; 392512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 392612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 392712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d %s slots_allocated %d\n", 392812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, 392912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, ppc440spe_chan->slots_allocated); 393012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 393112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 393212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* one is ok since we left it on there on purpose */ 393312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (in_use_descs > 1) 393412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n", 393512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin in_use_descs - 1); 393612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 393712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 393812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 39390793448187643b50af89d36b08470baf45a3cab4Linus Walleij * ppc440spe_adma_tx_status - poll the status of an ADMA transaction 394012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @chan: ADMA channel handle 394112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @cookie: ADMA transaction identifier 39420793448187643b50af89d36b08470baf45a3cab4Linus Walleij * @txstate: a holder for the current state of the channel 394312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 39440793448187643b50af89d36b08470baf45a3cab4Linus Walleijstatic enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan, 39450793448187643b50af89d36b08470baf45a3cab4Linus Walleij dma_cookie_t cookie, struct dma_tx_state *txstate) 394612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 394712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 394812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t last_used; 394912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t last_complete; 395012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin enum dma_status ret; 395112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 395212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 395312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin last_used = chan->cookie; 39544d4e58de32a192fea65ab84509d17d199bd291c8Russell King - ARM Linux last_complete = chan->completed_cookie; 395512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 3956bca3469205402d9fb14060d255d8786ae2256640Dan Williams dma_set_tx_state(txstate, last_complete, last_used, 0); 395712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 395812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = dma_async_is_complete(cookie, last_complete, last_used); 395912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret == DMA_SUCCESS) 396012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 396112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 396212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_slot_cleanup(ppc440spe_chan); 396312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 396412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin last_used = chan->cookie; 39654d4e58de32a192fea65ab84509d17d199bd291c8Russell King - ARM Linux last_complete = chan->completed_cookie; 396612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 3967bca3469205402d9fb14060d255d8786ae2256640Dan Williams dma_set_tx_state(txstate, last_complete, last_used, 0); 396812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 396912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return dma_async_is_complete(cookie, last_complete, last_used); 397012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 397112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 397212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 397312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_eot_handler - end of transfer interrupt handler 397412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 397512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data) 397612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 397712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan = data; 397812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 397912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, 398012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s\n", chan->device->id, __func__); 398112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 398212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_schedule(&chan->irq_tasklet); 398312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_device_clear_eot_status(chan); 398412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 398512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return IRQ_HANDLED; 398612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 398712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 398812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 398912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_err_handler - DMA error interrupt handler; 399012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * do the same things as a eot handler 399112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 399212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic irqreturn_t ppc440spe_adma_err_handler(int irq, void *data) 399312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 399412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan = data; 399512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 399612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, 399712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s\n", chan->device->id, __func__); 399812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 399912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_schedule(&chan->irq_tasklet); 400012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_device_clear_eot_status(chan); 400112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 400212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return IRQ_HANDLED; 400312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 400412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 400512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 400612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_test_callback - called when test operation has been done 400712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 400812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_test_callback(void *unused) 400912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 401012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin complete(&ppc440spe_r6_test_comp); 401112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 401212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 401312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 401412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w 401512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 401612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_issue_pending(struct dma_chan *chan) 401712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 401812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 401912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 402012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 402112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 402212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id, 402312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, ppc440spe_chan->pending); 402412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 402512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_chan->pending) { 402612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->pending = 0; 402712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_append(ppc440spe_chan); 402812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 402912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 403012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 403112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 403212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines 403312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * use FIFOs (as opposite to chains used in XOR) so this is a XOR 403412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * specific operation) 403512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 403612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan) 403712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 403812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 403912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t cookie; 404012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op; 404112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 404212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, 404312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s\n", chan->device->id, __func__); 404412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 404512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&chan->lock); 404612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op); 404712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op); 404812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 404912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 405012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_splice_init(&sw_desc->group_list, &chan->chain); 405112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin async_tx_ack(&sw_desc->async_tx); 405212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_null_xor(group_start); 405312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 405412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = chan->common.cookie; 405512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie++; 405612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cookie <= 1) 405712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = 2; 405812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 405912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* initialize the completed cookie to be less than 406012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the most recently used cookie 406112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 40624d4e58de32a192fea65ab84509d17d199bd291c8Russell King - ARM Linux chan->common.completed_cookie = cookie - 1; 406312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->common.cookie = sw_desc->async_tx.cookie = cookie; 406412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 406512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* channel should not be busy */ 406612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(ppc440spe_chan_is_busy(chan)); 406712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 406812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set the descriptor address */ 406912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc); 407012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 407112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* run the descriptor */ 407212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_run(chan); 407312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 407412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "ppc440spe adma%d" 407512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " failed to allocate null descriptor\n", 407612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id); 407712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 407812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 407912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 408012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 408112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully. 408212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * For this we just perform one WXOR operation with the same source 408312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * and destination addresses, the GF-multiplier is 1; so if RAID-6 408412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * capabilities are enabled then we'll get src/dst filled with zero. 408512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 408612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan) 408712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 408812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *iter; 408912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct page *pg; 409012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin char *a; 409112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t dma_addr, addrs[2]; 409212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 409312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int rval = 0; 409412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 409512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_WXOR, &op); 409612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 409712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pg = alloc_page(GFP_KERNEL); 409812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!pg) 409912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENOMEM; 410012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 410112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&chan->lock); 410212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1); 410312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 410412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 1 src, 1 dsr, int_ena, WXOR */ 410512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op); 410612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 410712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE); 410812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = PAGE_SIZE; 410912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 411012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 411112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rval = -EFAULT; 411212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 411312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto exit; 411412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 411512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 411612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 411712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Fill the test page with ones */ 411812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(page_address(pg), 0xFF, PAGE_SIZE); 411912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr = dma_map_page(chan->device->dev, pg, 0, 412012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 412112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 412212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup addresses */ 412312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0); 412412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0); 412512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addrs[0] = dma_addr; 412612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addrs[1] = 0; 412712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q); 412812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 412912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin async_tx_ack(&sw_desc->async_tx); 413012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.callback = ppc440spe_test_callback; 413112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.callback_param = NULL; 413212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 413312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin init_completion(&ppc440spe_r6_test_comp); 413412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 413512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_tx_submit(&sw_desc->async_tx); 413612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_issue_pending(&chan->common); 413712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 413812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin wait_for_completion(&ppc440spe_r6_test_comp); 413912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 414012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Now check if the test page is zeroed */ 414112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin a = page_address(pg); 414212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) { 414312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* page is zero - RAID-6 enabled */ 414412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rval = 0; 414512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 414612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RAID-6 was not enabled */ 414712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rval = -EINVAL; 414812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 414912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinexit: 415012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(pg); 415112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return rval; 415212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 415312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 415412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev) 415512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 415612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (adev->id) { 415712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 415812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 415912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_MEMCPY, adev->common.cap_mask); 416012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask); 416112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_MEMSET, adev->common.cap_mask); 416212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_PQ, adev->common.cap_mask); 416312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask); 416412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask); 416512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 416612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 416712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_XOR, adev->common.cap_mask); 416812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_PQ, adev->common.cap_mask); 416912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask); 417012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.cap_mask = adev->common.cap_mask; 417112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 417212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 417312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 417412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set base routines */ 417512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_alloc_chan_resources = 417612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_alloc_chan_resources; 417712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_free_chan_resources = 417812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_free_chan_resources; 41790793448187643b50af89d36b08470baf45a3cab4Linus Walleij adev->common.device_tx_status = ppc440spe_adma_tx_status; 418012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_issue_pending = ppc440spe_adma_issue_pending; 418112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 418212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set prep routines based on capability */ 418312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) { 418412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_memcpy = 418512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_memcpy; 418612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 418712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_MEMSET, adev->common.cap_mask)) { 418812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_memset = 418912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_memset; 419012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 419112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) { 419212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_xor = XOR_MAX_OPS; 419312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_xor = 419412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_xor; 419512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 419612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) { 419712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (adev->id) { 419812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 419912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_set_maxpq(&adev->common, 420012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0); 420112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 420212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 420312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_set_maxpq(&adev->common, 420412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0); 420512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 420612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 420712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_pq = XOR_MAX_OPS * 3; 420812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 420912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 421012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_pq = 421112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_pq; 421212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 421312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) { 421412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (adev->id) { 421512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 421612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_pq = DMA0_FIFO_SIZE / 421712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sizeof(struct dma_cdb); 421812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 421912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 422012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_pq = DMA1_FIFO_SIZE / 422112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sizeof(struct dma_cdb); 422212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 422312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 422412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_pq_val = 422512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_pqzero_sum; 422612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 422712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) { 422812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (adev->id) { 422912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 423012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_xor = DMA0_FIFO_SIZE / 423112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sizeof(struct dma_cdb); 423212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 423312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 423412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_xor = DMA1_FIFO_SIZE / 423512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sizeof(struct dma_cdb); 423612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 423712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 423812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_xor_val = 423912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_xor_zero_sum; 424012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 424112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) { 424212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_interrupt = 424312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_interrupt; 424412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 424512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: " 424612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "( %s%s%s%s%s%s%s)\n", 424712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_name(adev->dev), 424812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "", 424912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "", 425012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "", 425112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "", 425212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "", 425312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_MEMSET, adev->common.cap_mask) ? "memset " : "", 425412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : ""); 425512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 425612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 425712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev, 425812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 425912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int *initcode) 426012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 42612dc11581376829303b98eadb2de253bee065a56aGrant Likely struct platform_device *ofdev; 426212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct device_node *np; 426312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int ret; 426412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 42652dc11581376829303b98eadb2de253bee065a56aGrant Likely ofdev = container_of(adev->dev, struct platform_device, dev); 42663e6b02d9f5a9715f7d4ff9e0978e5f9cef53d31fDan Williams np = ofdev->dev.of_node; 426712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id != PPC440SPE_XOR_ID) { 426812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->err_irq = irq_of_parse_and_map(np, 1); 426912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->err_irq == NO_IRQ) { 427012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_warn(adev->dev, "no err irq resource?\n"); 427112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *initcode = PPC_ADMA_INIT_IRQ2; 427212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->err_irq = -ENXIO; 427312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 427412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin atomic_inc(&ppc440spe_adma_err_irq_ref); 427512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 427612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->err_irq = -ENXIO; 427712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 427812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 427912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->irq = irq_of_parse_and_map(np, 0); 428012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->irq == NO_IRQ) { 428112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(adev->dev, "no irq resource\n"); 428212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *initcode = PPC_ADMA_INIT_IRQ1; 428312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENXIO; 428412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_irq_map; 428512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 428612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(adev->dev, "irq %d, err irq %d\n", 428712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->irq, adev->err_irq); 428812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 428912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = request_irq(adev->irq, ppc440spe_adma_eot_handler, 429012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 0, dev_driver_string(adev->dev), chan); 429112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) { 429212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(adev->dev, "can't request irq %d\n", 429312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->irq); 429412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *initcode = PPC_ADMA_INIT_IRQ1; 429512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -EIO; 429612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_req1; 429712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 429812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 429912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* only DMA engines have a separate error IRQ 430012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * so it's Ok if err_irq < 0 in XOR engine case. 430112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 430212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->err_irq > 0) { 430312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* both DMA engines share common error IRQ */ 430412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = request_irq(adev->err_irq, 430512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_err_handler, 430612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin IRQF_SHARED, 430712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_driver_string(adev->dev), 430812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan); 430912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) { 431012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(adev->dev, "can't request irq %d\n", 431112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->err_irq); 431212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *initcode = PPC_ADMA_INIT_IRQ2; 431312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -EIO; 431412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_req2; 431512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 431612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 431712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 431812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) { 431912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* enable XOR engine interrupts */ 432012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT | 432112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT, 432212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &adev->xor_reg->ier); 432312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 432412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 mask, enable; 432512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 432612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe"); 432712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!np) { 432812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't find I2O device tree node\n", 432912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 433012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 433112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_req2; 433212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 433312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->i2o_reg = of_iomap(np, 0); 433412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!adev->i2o_reg) { 433512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to map I2O registers\n", __func__); 433612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 433712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -EINVAL; 433812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_req2; 433912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 434012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 434112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Unmask 'CS FIFO Attention' interrupts and 434212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * enable generating interrupts on errors 434312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 434412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin enable = (adev->id == PPC440SPE_DMA0_ID) ? 434512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) : 434612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM); 434712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mask = ioread32(&adev->i2o_reg->iopim) & enable; 434812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(mask, &adev->i2o_reg->iopim); 434912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 435012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 435112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 435212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_req2: 435312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin free_irq(adev->irq, chan); 435412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_req1: 435512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin irq_dispose_mapping(adev->irq); 435612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_irq_map: 435712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->err_irq > 0) { 435812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) 435912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin irq_dispose_mapping(adev->err_irq); 436012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 436112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 436212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 436312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 436412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev, 436512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 436612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 436712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 mask, disable; 436812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 436912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) { 437012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* disable XOR engine interrupts */ 437112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mask = ioread32be(&adev->xor_reg->ier); 437212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT | 437312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT); 437412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(mask, &adev->xor_reg->ier); 437512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 437612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* disable DMAx engine interrupts */ 437712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin disable = (adev->id == PPC440SPE_DMA0_ID) ? 437812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) : 437912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM); 438012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mask = ioread32(&adev->i2o_reg->iopim) | disable; 438112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(mask, &adev->i2o_reg->iopim); 438212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 438312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin free_irq(adev->irq, chan); 438412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin irq_dispose_mapping(adev->irq); 438512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->err_irq > 0) { 438612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin free_irq(adev->err_irq, chan); 438712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) { 438812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin irq_dispose_mapping(adev->err_irq); 438912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->i2o_reg); 439012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 439112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 439212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 439312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 439412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 439512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_probe - probe the asynch device 439612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 4397000061245a6797d542854106463b6b20fbdcb12eGrant Likelystatic int __devinit ppc440spe_adma_probe(struct platform_device *ofdev) 439812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 439905c02542c20aa00dc9a66f4bfb1a89d1131457f2Anatolij Gustschin struct device_node *np = ofdev->dev.of_node; 440012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct resource res; 440112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_device *adev; 440212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 440312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc_dma_chan_ref *ref, *_ref; 440412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int ret = 0, initcode = PPC_ADMA_INIT_OK; 440512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const u32 *idx; 440612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int len; 440712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin void *regs; 440812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 id, pool_size; 440912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 441012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (of_device_is_compatible(np, "amcc,xor-accelerator")) { 441112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin id = PPC440SPE_XOR_ID; 441212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* As far as the XOR engine is concerned, it does not 441312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * use FIFOs but uses linked list. So there is no dependency 441412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * between pool size to allocate and the engine configuration. 441512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 441612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pool_size = PAGE_SIZE << 1; 441712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 441812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* it is DMA0 or DMA1 */ 441912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idx = of_get_property(np, "cell-index", &len); 442012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!idx || (len != sizeof(u32))) { 442112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "Device node %s has missing " 442212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "or invalid cell-index property\n", 442312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np->full_name); 442412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 442512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 442612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin id = *idx; 442712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA0,1 engines use FIFO to maintain CDBs, so we 442812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * should allocate the pool accordingly to size of this 442912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * FIFO. Thus, the pool size depends on the FIFO depth: 443012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * how much CDBs pointers the FIFO may contain then so 443112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * much CDBs we should provide in the pool. 443212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * That is 443312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * CDB size = 32B; 443412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * CDBs number = (DMA0_FIFO_SIZE >> 3); 443512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Pool size = CDBs number * CDB size = 443612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2. 443712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 443812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pool_size = (id == PPC440SPE_DMA0_ID) ? 443912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA0_FIFO_SIZE : DMA1_FIFO_SIZE; 444012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pool_size <<= 2; 444112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 444212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 444312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (of_address_to_resource(np, 0, &res)) { 444412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to get memory resource\n"); 444512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_MEMRES; 444612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 444712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out; 444812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 444912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 445012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!request_mem_region(res.start, resource_size(&res), 445112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_driver_string(&ofdev->dev))) { 4452a584bff5efae8c1d026e3a930e3d13a90264fafcJoe Perches dev_err(&ofdev->dev, "failed to request memory region %pR\n", 4453a584bff5efae8c1d026e3a930e3d13a90264fafcJoe Perches &res); 445412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_MEMREG; 445512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -EBUSY; 445612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out; 445712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 445812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 445912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* create a device */ 446012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev = kzalloc(sizeof(*adev), GFP_KERNEL); 446112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!adev) { 446212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to allocate device\n"); 446312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_ALLOC; 446412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 446512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_adev_alloc; 446612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 446712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 446812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->id = id; 446912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->pool_size = pool_size; 447012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* allocate coherent memory for hardware descriptors */ 447112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev, 447212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->pool_size, &adev->dma_desc_pool, 447312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin GFP_KERNEL); 447412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->dma_desc_pool_virt == NULL) { 447512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent " 447612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "memory for hardware descriptors\n", 447712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->pool_size); 447812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_COHERENT; 447912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 448012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_dma_alloc; 448112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 448212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(&ofdev->dev, "allocted descriptor pool virt 0x%p phys 0x%llx\n", 448312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool); 448412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 448512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin regs = ioremap(res.start, resource_size(&res)); 448612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!regs) { 448712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to ioremap regs!\n"); 448812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_regs_alloc; 448912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 449012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 449112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) { 449212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->xor_reg = regs; 449312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Reset XOR */ 449412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr); 449512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr); 449612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 449712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ? 449812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA0_FIFO_SIZE : DMA1_FIFO_SIZE; 449912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_reg = regs; 450012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMAx_FIFO_SIZE is defined in bytes, 450112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * <fsiz> - is defined in number of CDB pointers (8byte). 450212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * DMA FIFO Length = CSlength + CPlength, where 450312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * CSlength = CPlength = (fsiz + 1) * 8. 450412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 450512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2), 450612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &adev->dma_reg->fsiz); 450712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Configure DMA engine */ 450812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN, 450912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &adev->dma_reg->cfg); 451012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Clear Status */ 451112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(~0, &adev->dma_reg->dsts); 451212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 451312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 451412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dev = &ofdev->dev; 451512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.dev = &ofdev->dev; 451612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&adev->common.channels); 451712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_set_drvdata(&ofdev->dev, adev); 451812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 451912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* create a channel */ 452012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = kzalloc(sizeof(*chan), GFP_KERNEL); 452112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!chan) { 452212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "can't allocate channel structure\n"); 452312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_CHANNEL; 452412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 452512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_chan_alloc; 452612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 452712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 452812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_init(&chan->lock); 452912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&chan->chain); 453012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&chan->all_slots); 453112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device = adev; 453212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->common.device = &adev->common; 453312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_add_tail(&chan->common.device_node, &adev->common.channels); 453412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_init(&chan->irq_tasklet, ppc440spe_adma_tasklet, 453512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (unsigned long)chan); 453612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 453712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* allocate and map helper pages for async validation or 453812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * async_mult/async_sum_product operations on DMA0/1. 453912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 454012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id != PPC440SPE_XOR_ID) { 454112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->pdest_page = alloc_page(GFP_KERNEL); 454212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->qdest_page = alloc_page(GFP_KERNEL); 454312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!chan->pdest_page || 454412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin !chan->qdest_page) { 454512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan->pdest_page) 454612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(chan->pdest_page); 454712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan->qdest_page) 454812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(chan->qdest_page); 454912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 455012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_page_alloc; 455112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 455212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0, 455312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 455412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0, 455512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 455612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 455712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 455812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ref = kmalloc(sizeof(*ref), GFP_KERNEL); 455912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ref) { 456012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ref->chan = &chan->common; 456112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&ref->node); 456212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_add_tail(&ref->node, &ppc440spe_adma_chan_list); 456312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 456412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to allocate channel reference!\n"); 456512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 456612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_ref_alloc; 456712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 456812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 456912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode); 457012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) 457112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_irq; 457212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 457312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_init_capabilities(adev); 457412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 457512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = dma_async_device_register(&adev->common); 457612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) { 457712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_REGISTER; 457812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to register dma device\n"); 457912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_dev_reg; 458012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 458112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 458212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out; 458312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 458412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_dev_reg: 458512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_release_irqs(adev, chan); 458612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_irq: 458712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) { 458812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan == to_ppc440spe_adma_chan(ref->chan)) { 458912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&ref->node); 459012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ref); 459112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 459212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 459312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_ref_alloc: 459412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id != PPC440SPE_XOR_ID) { 459512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(&ofdev->dev, chan->pdest, 459612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 459712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(&ofdev->dev, chan->qdest, 459812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 459912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(chan->pdest_page); 460012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(chan->qdest_page); 460112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 460212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_page_alloc: 460312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(chan); 460412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_chan_alloc: 460512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) 460612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->xor_reg); 460712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 460812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->dma_reg); 460912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_regs_alloc: 461012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_free_coherent(adev->dev, adev->pool_size, 461112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool_virt, 461212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool); 461312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_dma_alloc: 461412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(adev); 461512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_adev_alloc: 461612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin release_mem_region(res.start, resource_size(&res)); 461712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout: 461812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (id < PPC440SPE_ADMA_ENGINES_NUM) 461912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_devices[id] = initcode; 462012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 462112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 462212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 462312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 462412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 462512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_remove - remove the asynch device 462612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 46272dc11581376829303b98eadb2de253bee065a56aGrant Likelystatic int __devexit ppc440spe_adma_remove(struct platform_device *ofdev) 462812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 462912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_device *adev = dev_get_drvdata(&ofdev->dev); 463005c02542c20aa00dc9a66f4bfb1a89d1131457f2Anatolij Gustschin struct device_node *np = ofdev->dev.of_node; 463112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct resource res; 463212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, *_chan; 463312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc_dma_chan_ref *ref, *_ref; 463412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 463512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 463612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_set_drvdata(&ofdev->dev, NULL); 463712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id < PPC440SPE_ADMA_ENGINES_NUM) 463812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_devices[adev->id] = -1; 463912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 464012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_async_device_unregister(&adev->common); 464112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 464212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(chan, _chan, &adev->common.channels, 464312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin device_node) { 464412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 464512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_release_irqs(adev, ppc440spe_chan); 464612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_kill(&ppc440spe_chan->irq_tasklet); 464712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id != PPC440SPE_XOR_ID) { 464812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest, 464912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 465012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest, 465112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 465212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(ppc440spe_chan->pdest_page); 465312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(ppc440spe_chan->qdest_page); 465412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 465512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, 465612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin node) { 465712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_chan == 465812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin to_ppc440spe_adma_chan(ref->chan)) { 465912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&ref->node); 466012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ref); 466112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 466212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 466312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&chan->device_node); 466412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ppc440spe_chan); 466512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 466612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 466712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_free_coherent(adev->dev, adev->pool_size, 466812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool_virt, adev->dma_desc_pool); 466912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) 467012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->xor_reg); 467112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 467212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->dma_reg); 467312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_address_to_resource(np, 0, &res); 467412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin release_mem_region(res.start, resource_size(&res)); 467512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(adev); 467612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 467712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 467812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 467912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* 468012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * /sys driver interface to enable h/w RAID-6 capabilities 468112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/ 468212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * directory are "devices", "enable" and "poly". 468312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * "devices" shows available engines. 468412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * "enable" is used to enable RAID-6 capabilities or to check 468512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * whether these has been activated. 468612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * "poly" allows setting/checking used polynomial (for PPC440SPe only). 468712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 468812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 468912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t show_ppc440spe_devices(struct device_driver *dev, char *buf) 469012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 469112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ssize_t size = 0; 469212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 469312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 469412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) { 469512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_adma_devices[i] == -1) 469612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 469712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size += snprintf(buf + size, PAGE_SIZE - size, 469812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "PPC440SP(E)-ADMA.%d: %s\n", i, 469912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc_adma_errors[ppc440spe_adma_devices[i]]); 470012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 470112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return size; 470212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 470312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 470412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t show_ppc440spe_r6enable(struct device_driver *dev, char *buf) 470512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 470612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return snprintf(buf, PAGE_SIZE, 470712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "PPC440SP(e) RAID-6 capabilities are %sABLED.\n", 470812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_r6_enabled ? "EN" : "DIS"); 470912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 471012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 471112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t store_ppc440spe_r6enable(struct device_driver *dev, 471212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const char *buf, size_t count) 471312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 471412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long val; 471512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 471612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!count || count > 11) 471712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 471812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 471912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!ppc440spe_r6_tchan) 472012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EFAULT; 472112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 472212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Write a key */ 472312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sscanf(buf, "%lx", &val); 472412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val); 472512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin isync(); 472612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 472712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Verify whether it really works now */ 472812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) { 472912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_info("PPC440SP(e) RAID-6 has been activated " 473012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "successfully\n"); 473112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_r6_enabled = 1; 473212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 473312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_info("PPC440SP(e) RAID-6 hasn't been activated!" 473412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " Error key ?\n"); 473512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_r6_enabled = 0; 473612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 473712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return count; 473812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 473912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 474012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t show_ppc440spe_r6poly(struct device_driver *dev, char *buf) 474112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 474212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ssize_t size = 0; 474312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 reg; 474412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 474512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#ifdef CONFIG_440SP 474612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 440SP has fixed polynomial */ 474712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg = 0x4d; 474812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#else 474912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL); 475012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg >>= MQ0_CFBHL_POLY; 475112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg &= 0xFF; 475212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#endif 475312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 475412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver " 475512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "uses 0x1%02x polynomial.\n", reg); 475612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return size; 475712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 475812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 475912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t store_ppc440spe_r6poly(struct device_driver *dev, 476012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const char *buf, size_t count) 476112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 476212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long reg, val; 476312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 476412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#ifdef CONFIG_440SP 476512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 440SP uses default 0x14D polynomial only */ 476612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 476712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#endif 476812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 476912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!count || count > 6) 477012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 477112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 477212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* e.g., 0x14D or 0x11D */ 477312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sscanf(buf, "%lx", &val); 477412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 477512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (val & ~0x1FF) 477612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 477712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 477812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin val &= 0xFF; 477912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL); 478012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg &= ~(0xFF << MQ0_CFBHL_POLY); 478112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg |= val << MQ0_CFBHL_POLY; 478212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg); 478312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 478412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return count; 478512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 478612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 478712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic DRIVER_ATTR(devices, S_IRUGO, show_ppc440spe_devices, NULL); 478812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic DRIVER_ATTR(enable, S_IRUGO | S_IWUSR, show_ppc440spe_r6enable, 478912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin store_ppc440spe_r6enable); 479012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic DRIVER_ATTR(poly, S_IRUGO | S_IWUSR, show_ppc440spe_r6poly, 479112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin store_ppc440spe_r6poly); 479212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 479312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* 479412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Common initialisation for RAID engines; allocate memory for 479512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * DMAx FIFOs, perform configuration common for all DMA engines. 479612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Further DMA engine specific configuration is done at probe time. 479712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 479812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_configure_raid_devices(void) 479912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 480012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct device_node *np; 480112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct resource i2o_res; 480212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct i2o_regs __iomem *i2o_reg; 480312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_host_t i2o_dcr_host; 480412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int dcr_base, dcr_len; 480512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i, ret; 480612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 480712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe"); 480812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!np) { 480912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't find I2O device tree node\n", 481012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 481112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENODEV; 481212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 481312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 481412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (of_address_to_resource(np, 0, &i2o_res)) { 481512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 481612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 481712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 481812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 481912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i2o_reg = of_iomap(np, 0); 482012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!i2o_reg) { 482112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to map I2O registers\n", __func__); 482212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 482312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 482412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 482512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 482612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Get I2O DCRs base */ 482712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_base = dcr_resource_start(np, 0); 482812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_len = dcr_resource_len(np, 0); 482912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!dcr_base && !dcr_len) { 483012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't get DCR registers base/len!\n", 483112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np->full_name); 483212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 483312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(i2o_reg); 483412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENODEV; 483512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 483612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 483712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i2o_dcr_host = dcr_map(np, dcr_base, dcr_len); 483812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!DCR_MAP_OK(i2o_dcr_host)) { 483912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to map DCRs!\n", np->full_name); 484012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 484112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(i2o_reg); 484212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENODEV; 484312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 484412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 484512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 484612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share 484712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the base address of FIFO memory space. 484812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Actually we need twice more physical memory than programmed in the 484912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * <fsiz> register (because there are two FIFOs for each DMA: CP and CS) 485012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 485112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1, 485212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin GFP_KERNEL); 485312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!ppc440spe_dma_fifo_buf) { 485412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__); 485512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(i2o_reg); 485612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_unmap(i2o_dcr_host, dcr_len); 485712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENOMEM; 485812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 485912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 486012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 486112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Configure h/w 486212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 486312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Reset I2O/DMA */ 486412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA); 486512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mtdcri(SDR0, DCRN_SDR0_SRST, 0); 486612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 486712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup the base address of mmaped registers */ 486812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32)); 486912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) | 487012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin I2O_REG_ENABLE); 487112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_unmap(i2o_dcr_host, dcr_len); 487212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 487312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup FIFO memory space base address */ 487412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(0, &i2o_reg->ifbah); 487512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal); 487612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 487712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set zero FIFO size for I2O, so the whole 487812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma_fifo_buf is used by DMAs. 487912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * DMAx_FIFOs will be configured while probe. 488012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 488112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(0, &i2o_reg->ifsiz); 488212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(i2o_reg); 488312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 488412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* To prepare WXOR/RXOR functionality we need access to 488512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Memory Queue Module DCRs (finally it will be enabled 488612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * via /sys interface of the ppc440spe ADMA driver). 488712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 488812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe"); 488912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!np) { 489012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't find MQ device tree node\n", 489112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 489212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 489312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_free; 489412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 489512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 489612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Get MQ DCRs base */ 489712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_base = dcr_resource_start(np, 0); 489812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_len = dcr_resource_len(np, 0); 489912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!dcr_base && !dcr_len) { 490012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't get DCR registers base/len!\n", 490112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np->full_name); 490212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 490312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_mq; 490412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 490512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 490612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len); 490712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) { 490812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to map DCRs!\n", np->full_name); 490912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 491012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_mq; 491112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 491212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 491312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_mq_dcr_len = dcr_len; 491412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 491512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set HB alias */ 491612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB); 491712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 491812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set: 491912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - LL transaction passing limit to 1; 492012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - Memory controller cycle limit to 1; 492112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - Galois Polynomial to 0x14d (default) 492212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 492312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, 492412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) | 492512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY)); 492612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 492712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin atomic_set(&ppc440spe_adma_err_irq_ref, 0); 492812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) 492912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_devices[i] = -1; 493012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 493112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 493212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 493312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_mq: 493412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 493512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_free: 493612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ppc440spe_dma_fifo_buf); 493712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 493812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 493912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 49404b1cf1facca31b7db2a61d8aa2ba40d5a93a0957Márton Némethstatic const struct of_device_id ppc440spe_adma_of_match[] __devinitconst = { 494112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin { .compatible = "ibm,dma-440spe", }, 494212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin { .compatible = "amcc,xor-accelerator", }, 494312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin {}, 494412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 494512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinMODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match); 494612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 4947000061245a6797d542854106463b6b20fbdcb12eGrant Likelystatic struct platform_driver ppc440spe_adma_driver = { 494812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin .probe = ppc440spe_adma_probe, 494912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin .remove = __devexit_p(ppc440spe_adma_remove), 495012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin .driver = { 495112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin .name = "PPC440SP(E)-ADMA", 495212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin .owner = THIS_MODULE, 49534018294b53d1dae026880e45f174c1cc63b5d435Grant Likely .of_match_table = ppc440spe_adma_of_match, 495412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin }, 495512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 495612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 495712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic __init int ppc440spe_adma_init(void) 495812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 495912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int ret; 496012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 496112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = ppc440spe_configure_raid_devices(); 496212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) 496312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 496412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 4965000061245a6797d542854106463b6b20fbdcb12eGrant Likely ret = platform_driver_register(&ppc440spe_adma_driver); 496612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) { 496712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to register platform driver\n", 496812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 496912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_reg; 497012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 497112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 497212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Initialization status */ 497312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = driver_create_file(&ppc440spe_adma_driver.driver, 497412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_devices); 497512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) 497612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_dev; 497712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 497812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RAID-6 h/w enable entry */ 497912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = driver_create_file(&ppc440spe_adma_driver.driver, 498012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_enable); 498112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) 498212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_en; 498312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 498412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* GF polynomial to use */ 498512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = driver_create_file(&ppc440spe_adma_driver.driver, 498612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_poly); 498712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!ret) 498812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 498912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 499012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 499112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_enable); 499212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_en: 499312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 499412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_devices); 499512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_dev: 499612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* User will not be able to enable h/w RAID-6 */ 499712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to create RAID-6 driver interface\n", 499812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 4999000061245a6797d542854106463b6b20fbdcb12eGrant Likely platform_driver_unregister(&ppc440spe_adma_driver); 500012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_reg: 500112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len); 500212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ppc440spe_dma_fifo_buf); 500312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 500412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 500512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 500612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void __exit ppc440spe_adma_exit(void) 500712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 500812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 500912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_poly); 501012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 501112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_enable); 501212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 501312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_devices); 5014000061245a6797d542854106463b6b20fbdcb12eGrant Likely platform_driver_unregister(&ppc440spe_adma_driver); 501512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len); 501612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ppc440spe_dma_fifo_buf); 501712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 501812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 501912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinarch_initcall(ppc440spe_adma_init); 502012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinmodule_exit(ppc440spe_adma_exit); 502112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 502212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinMODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>"); 502312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinMODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver"); 502412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinMODULE_LICENSE("GPL"); 5025