adma.c revision 427cdf19b97e509e21e5d347e18d8b0b34723dfc
112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* 212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Copyright (C) 2006-2009 DENX Software Engineering. 312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Author: Yuri Tikhonov <yur@emcraft.com> 512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Further porting to arch/powerpc by 712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Anatolij Gustschin <agust@denx.de> 812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * This program is free software; you can redistribute it and/or modify it 1012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * under the terms of the GNU General Public License as published by the Free 1112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Software Foundation; either version 2 of the License, or (at your option) 1212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * any later version. 1312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 1412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * This program is distributed in the hope that it will be useful, but WITHOUT 1512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 1612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 1712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * more details. 1812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 1912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * You should have received a copy of the GNU General Public License along with 2012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * this program; if not, write to the Free Software Foundation, Inc., 59 2112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Temple Place - Suite 330, Boston, MA 02111-1307, USA. 2212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 2312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * The full GNU General Public License is included in this distribution in the 2412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * file called COPYING. 2512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 2612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 2712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* 2812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * This driver supports the asynchrounous DMA copy and RAID engines available 2912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * on the AMCC PPC440SPe Processors. 3012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) 3112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ADMA driver written by D.Williams. 3212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 3312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 3412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/init.h> 3512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/module.h> 3612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/async_tx.h> 3712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/delay.h> 3812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/dma-mapping.h> 3912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/spinlock.h> 4012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/interrupt.h> 415a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 4212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/uaccess.h> 4312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/proc_fs.h> 4412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/of.h> 4512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/of_platform.h> 4612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <asm/dcr.h> 4712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <asm/dcr-regs.h> 4812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include "adma.h" 4912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 5012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinenum ppc_adma_init_code { 5112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_OK = 0, 5212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_MEMRES, 5312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_MEMREG, 5412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_ALLOC, 5512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_COHERENT, 5612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_CHANNEL, 5712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_IRQ1, 5812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_IRQ2, 5912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_REGISTER 6012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 6112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 6212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic char *ppc_adma_errors[] = { 6312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_OK] = "ok", 6412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource", 6512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_MEMREG] = "failed to request memory region", 6612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev " 6712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "structure", 6812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for " 6912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "hardware descriptors", 7012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel", 7112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_IRQ1] = "failed to request first irq", 7212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_IRQ2] = "failed to request second irq", 7312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device", 7412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 7512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 7612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic enum ppc_adma_init_code 7712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM]; 7812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 7912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstruct ppc_dma_chan_ref { 8012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan; 8112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct list_head node; 8212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 8312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 8412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* The list of channels exported by ppc440spe ADMA */ 8512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstruct list_head 8612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list); 8712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 8812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* This flag is set when want to refetch the xor chain in the interrupt 8912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * handler 9012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 9112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 do_xor_refetch; 9212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 9312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* Pointer to DMA0, DMA1 CP/CS FIFO */ 9412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void *ppc440spe_dma_fifo_buf; 9512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 9612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* Pointers to last submitted to DMA0, DMA1 CDBs */ 9712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *chan_last_sub[3]; 9812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *chan_first_cdb[3]; 9912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 10012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* Pointer to last linked and submitted xor CB */ 10112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *xor_last_linked; 10212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *xor_last_submit; 10312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 10412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* This array is used in data-check operations for storing a pattern */ 10512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic char ppc440spe_qword[16]; 10612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 10712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic atomic_t ppc440spe_adma_err_irq_ref; 10812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic dcr_host_t ppc440spe_mq_dcr_host; 10912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic unsigned int ppc440spe_mq_dcr_len; 11012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 11112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* Since RXOR operations use the common register (MQ0_CF2H) for setting-up 11212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the block size in transactions, then we do not allow to activate more than 11312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * only one RXOR transactions simultaneously. So use this var to store 11412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is 11512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * set) or not (PPC440SPE_RXOR_RUN is clear). 11612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 11712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic unsigned long ppc440spe_rxor_state; 11812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 11912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* These are used in enable & check routines 12012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 12112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 ppc440spe_r6_enabled; 12212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_chan *ppc440spe_r6_tchan; 12312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct completion ppc440spe_r6_test_comp; 12412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 12512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_dma2rxor_prep_src( 12612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 12712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_rxor *cursor, int index, 12812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, u32 addr); 12912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_set_src( 13012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 13112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index, dma_addr_t addr); 13212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_set_mult( 13312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 13412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index, u8 mult); 13512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 13612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#ifdef ADMA_LL_DEBUG 13712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define ADMA_LL_DBG(x) ({ if (1) x; 0; }) 13812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#else 13912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define ADMA_LL_DBG(x) ({ if (0) x; 0; }) 14012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#endif 14112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 14212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void print_cb(struct ppc440spe_adma_chan *chan, void *block) 14312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 14412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *cdb; 14512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *cb; 14612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 14712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 14812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 14912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 0: 15012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 1: 15112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb = block; 15212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 15312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("CDB at %p [%d]:\n" 15412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n" 15512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t sg1u 0x%08x sg1l 0x%08x\n" 15612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t sg2u 0x%08x sg2l 0x%08x\n" 15712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t sg3u 0x%08x sg3l 0x%08x\n", 15812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb, chan->device->id, 15912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt), 16012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l), 16112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l), 16212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l) 16312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ); 16412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 16512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 2: 16612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb = block; 16712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 16812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("CB at %p [%d]:\n" 16912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n" 17012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t cbtah 0x%08x cbtal 0x%08x\n" 17112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t cblah 0x%08x cblal 0x%08x\n", 17212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb, chan->device->id, 17312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb->cbc, cb->cbbc, cb->cbs, 17412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb->cbtah, cb->cbtal, 17512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb->cblah, cb->cblal); 17612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < 16; i++) { 17712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i && !cb->ops[i].h && !cb->ops[i].l) 17812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 17912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n", 18012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i, cb->ops[i].h, cb->ops[i].l); 18112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 18212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 18312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 18412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 18512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 18612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void print_cb_list(struct ppc440spe_adma_chan *chan, 18712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter) 18812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 18912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (; iter; iter = iter->hw_next) 19012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin print_cb(chan, iter->hw_desc); 19112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 19212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 19312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src, 19412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt) 19512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 19612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 19712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 19812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\n%s(%d):\nsrc: ", __func__, id); 19912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 20012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx ", src[i]); 20112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("dst:\n\t0x%016llx\n", dst); 20212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 20312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 20412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src, 20512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt) 20612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 20712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 20812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 20912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\n%s(%d):\nsrc: ", __func__, id); 21012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 21112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx ", src[i]); 21212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("dst: "); 21312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < 2; i++) 21412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx ", dst[i]); 21512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 21612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 21712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src, 21812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt, 21912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf) 22012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 22112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 22212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 22312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id); 22412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (scf) { 22512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 22612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]); 22712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 22812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 22912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx(no) ", src[i]); 23012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 23112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 23212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("dst: "); 23312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < 2; i++) 23412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx ", src[src_cnt + i]); 23512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 23612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 23712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/****************************************************************************** 23812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Command (Descriptor) Blocks low-level routines 23912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ******************************************************************************/ 24012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 24112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT 24212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * pseudo operation 24312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 24412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc, 24512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 24612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 24712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *p; 24812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 24912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 25012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 25112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin p = desc->hw_desc; 25212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 25312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* NOP with Command Block Complete Enable */ 25412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin p->cbc = XOR_CBCR_CBCE_BIT; 25512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 25612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 25712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 25812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 25912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* NOP with interrupt */ 26012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &desc->flags); 26112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 26212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 26312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id, 26412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 26512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 26612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 26712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 26812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 26912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 27012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR 27112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * pseudo operation 27212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 27312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc) 27412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 27512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 27612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 27712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = 0; 27812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = 1; 27912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 28012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 28112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 28212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation 28312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 28412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc, 28512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, unsigned long flags) 28612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 28712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *hw_desc = desc->hw_desc; 28812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 28912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 29012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 29112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = src_cnt; 29212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = 1; 29312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 29412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt; 29512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 29612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Enable interrupt on completion */ 29712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc |= XOR_CBCR_CBCE_BIT; 29812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 29912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 30012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 30112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ 30212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * operation in DMA2 controller 30312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 30412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc, 30512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int dst_cnt, int src_cnt, unsigned long flags) 30612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 30712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *hw_desc = desc->hw_desc; 30812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 30912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 31012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 31112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = src_cnt; 31212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = dst_cnt; 31312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags)); 31412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->descs_per_op = 0; 31512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 31612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc = XOR_CBCR_TGT_BIT; 31712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 31812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Enable interrupt on completion */ 31912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc |= XOR_CBCR_CBCE_BIT; 32012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 32112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 32212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define DMA_CTRL_FLAGS_LAST DMA_PREP_FENCE 32312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define DMA_PREP_ZERO_P (DMA_CTRL_FLAGS_LAST << 1) 32412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define DMA_PREP_ZERO_Q (DMA_PREP_ZERO_P << 1) 32512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 32612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 32712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation 32812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * with DMA0/1 32912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 33012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc, 33112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int dst_cnt, int src_cnt, unsigned long flags, 33212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op) 33312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 33412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 33512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 33612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 dopc; 33712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 33812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Common initialization of a PQ descriptors chain */ 33912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bits(op, &desc->flags); 34012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = src_cnt; 34112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = dst_cnt; 34212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 34312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR MULTICAST if both P and Q are being computed 34412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * MV_SG1_SG2 if Q only 34512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 34612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ? 34712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2; 34812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 34912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &desc->group_list, chain_node) { 35012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 35112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 35212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 35312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(!list_is_last(&iter->chain_node, 35412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &desc->group_list))) { 35512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 35612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 35712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 35812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 35912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 36012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is the last descriptor. 36112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * this slot will be pasted from ADMA level 36212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * each time it wants to configure parameters 36312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * of the transaction (src, dst, ...) 36412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 36512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 36612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 36712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 36812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 36912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 37012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 37112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 37212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 37312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set OPS depending on WXOR/RXOR type of operation */ 37412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) { 37512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is a WXOR only chain: 37612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - first descriptors are for zeroing destinations 37712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * if PPC440SPE_ZERO_P/Q set; 37812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - descriptors remained are for GF-XOR operations. 37912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 38012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&desc->group_list, 38112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 38212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 38312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 38412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) { 38512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 38612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 38712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 38812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 38912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 39012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 39112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 39212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) { 39312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 39412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 39512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 39612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 39712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 39812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 39912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 40012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &desc->group_list, chain_node) { 40112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 40212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = dopc; 40312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 40412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 40512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is either RXOR-only or mixed RXOR/WXOR */ 40612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 40712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* The first 1 or 2 slots in chain are always RXOR, 40812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * if need to calculate P & Q, then there are two 40912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * RXOR slots; if only P or only Q, then there is one 41012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 41112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&desc->group_list, 41212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 41312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 41412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 41512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 41612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 41712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (desc->dst_cnt == DMA_DEST_MAX_NUM) { 41812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 41912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 42012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 42112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 42212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 42312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 42412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 42512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* The remaining descs (if any) are WXORs */ 42612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) { 42712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 42812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 42912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 43012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &desc->group_list, 43112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 43212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 43312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = dopc; 43412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 43512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 43612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 43712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 43812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 43912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 44012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor 44112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for PQ_ZERO_SUM operation 44212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 44312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_dma01pqzero_sum( 44412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 44512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int dst_cnt, int src_cnt) 44612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 44712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 44812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 44912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i = 0; 45012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST : 45112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_OPC_MV_SG1_SG2; 45212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 45312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Initialize starting from 2nd or 3rd descriptor dependent 45412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * on dst_cnt. First one or two slots are for cloning P 45512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * and/or Q to chan->pdest and/or chan->qdest as we have 45612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * to preserve original P/Q. 45712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 45812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&desc->group_list, 45912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 46012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 46112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 46212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 46312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dst_cnt > 1) { 46412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 46512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 46612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 46712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* initialize each source descriptor in chain */ 46812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &desc->group_list, chain_node) { 46912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 47012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 47112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->src_cnt = 0; 47212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->dst_cnt = 0; 47312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 47412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is a ZERO_SUM operation: 47512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - <src_cnt> descriptors starting from 2nd or 3rd 47612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor are for GF-XOR operations; 47712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - remaining <dst_cnt> descriptors are for checking the result 47812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 47912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i++ < src_cnt) 48012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* MV_SG1_SG2 if only Q is being verified 48112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * MULTICAST if both P and Q are being verified 48212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 48312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = dopc; 48412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 48512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA_CDB_OPC_DCHECK128 operation */ 48612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_DCHECK128; 48712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 48812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(!list_is_last(&iter->chain_node, 48912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &desc->group_list))) { 49012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 49112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 49212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 49312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 49412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 49512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is the last descriptor. 49612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * this slot will be pasted from ADMA level 49712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * each time it wants to configure parameters 49812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * of the transaction (src, dst, ...) 49912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 50012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 50112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* always enable interrupt generation since we get 50212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the status of pqzero from the handler 50312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 50412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 50512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 50612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 50712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = src_cnt; 50812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = dst_cnt; 50912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 51012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 51112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 51212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation 51312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 51412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc, 51512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long flags) 51612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 51712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc = desc->hw_desc; 51812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 51912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 52012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 52112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = 1; 52212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = 1; 52312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 52412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 52512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &desc->flags); 52612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 52712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &desc->flags); 52812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 52912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 53012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 53112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 53212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 53312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_memset - initialize the descriptor for MEMSET operation 53412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 53512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_memset(struct ppc440spe_adma_desc_slot *desc, 53612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int value, unsigned long flags) 53712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 53812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc = desc->hw_desc; 53912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 54012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 54112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 54212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = 1; 54312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = 1; 54412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 54512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 54612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &desc->flags); 54712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 54812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &desc->flags); 54912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 55012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->sg1u = hw_desc->sg1l = cpu_to_le32((u32)value); 55112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->sg3u = hw_desc->sg3l = cpu_to_le32((u32)value); 55212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_DFILL128; 55312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 55412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 55512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 55612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_src_addr - set source address into the descriptor 55712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 55812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc, 55912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 56012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_idx, dma_addr_t addrh, 56112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addrl) 56212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 56312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 56412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 56512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin phys_addr_t addr64, tmplow, tmphi; 56612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 56712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 56812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 56912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 57012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addrh) { 57112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr64 = addrl; 57212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmphi = (addr64 >> 32); 57312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmplow = (addr64 & 0xFFFFFFFF); 57412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 57512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmphi = addrh; 57612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmplow = addrl; 57712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 57812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 57912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow); 58012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi); 58112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 58212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 58312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 58412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->ops[src_idx].l = addrl; 58512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->ops[src_idx].h |= addrh; 58612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 58712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 58812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 58912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 59012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 59112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_src_mult - set source address mult into the descriptor 59212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 59312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc, 59412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, u32 mult_index, 59512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int sg_index, unsigned char mult_value) 59612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 59712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 59812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 59912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 *psgu; 60012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 60112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 60212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 60312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 60412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 60512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 60612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (sg_index) { 60712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* for RXOR operations set multiplier 60812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * into source cued address 60912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 61012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_SG_SRC: 61112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgu = &dma_hw_desc->sg1u; 61212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 61312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* for WXOR operations set multiplier 61412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * into destination cued address(es) 61512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 61612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_SG_DST1: 61712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgu = &dma_hw_desc->sg2u; 61812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 61912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_SG_DST2: 62012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgu = &dma_hw_desc->sg3u; 62112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 62212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 62312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 62412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 62512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 62612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *psgu |= cpu_to_le32(mult_value << mult_index); 62712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 62812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 62912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 63012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 63112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 63212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 63312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 63412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 63512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 63612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 63712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_dest_addr - set destination address into the descriptor 63812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 63912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc, 64012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 64112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addrh, dma_addr_t addrl, 64212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 dst_idx) 64312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 64412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 64512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 64612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin phys_addr_t addr64, tmphi, tmplow; 64712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 *psgu, *psgl; 64812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 64912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 65012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 65112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 65212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addrh) { 65312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr64 = addrl; 65412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmphi = (addr64 >> 32); 65512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmplow = (addr64 & 0xFFFFFFFF); 65612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 65712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmphi = addrh; 65812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmplow = addrl; 65912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 66012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 66112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 66212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u; 66312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l; 66412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 66512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *psgl = cpu_to_le32((u32)tmplow); 66612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *psgu |= cpu_to_le32((u32)tmphi); 66712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 66812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 66912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 67012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbtal = addrl; 67112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbtah |= addrh; 67212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 67312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 67412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 67512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 67612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 67712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_byte_count - set number of data bytes involved 67812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * into the operation 67912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 68012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc, 68112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 68212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 byte_count) 68312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 68412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 68512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 68612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 68712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 68812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 68912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 69012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 69112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->cnt = cpu_to_le32(byte_count); 69212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 69312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 69412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 69512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbbc = byte_count; 69612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 69712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 69812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 69912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 70012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 70112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_rxor_block_size - set RXOR block size 70212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 70312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count) 70412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 70512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* assume that byte_count is aligned on the 512-boundary; 70612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * thus write it directly to the register (bits 23:31 are 70712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * reserved there). 70812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 70912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count); 71012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 71112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 71212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 71312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_dcheck - set CHECK pattern 71412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 71512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc, 71612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, u8 *qword) 71712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 71812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 71912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 72012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 72112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 72212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 72312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 72412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(qword[0], &dma_hw_desc->sg3l); 72512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(qword[4], &dma_hw_desc->sg3u); 72612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(qword[8], &dma_hw_desc->sg2l); 72712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(qword[12], &dma_hw_desc->sg2u); 72812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 72912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 73012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 73112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 73212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 73312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 73412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 73512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_xor_set_link - set link address in xor CB 73612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 73712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc, 73812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *next_desc) 73912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 74012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc = prev_desc->hw_desc; 74112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 74212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!next_desc || !(next_desc->phys))) { 74312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n", 74412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, next_desc, 74512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin next_desc ? next_desc->phys : 0); 74612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 74712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 74812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 74912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbs = 0; 75012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cblal = next_desc->phys; 75112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cblah = 0; 75212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT; 75312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 75412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 75512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 75612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_link - set the address of descriptor following this 75712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor in chain 75812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 75912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan, 76012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *prev_desc, 76112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *next_desc) 76212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 76312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long flags; 76412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *tail = next_desc; 76512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 76612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!prev_desc || !next_desc || 76712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (prev_desc->hw_next && prev_desc->hw_next != next_desc))) { 76812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* If previous next is overwritten something is wrong. 76912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * though we may refetch from append to initiate list 77012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * processing; in this case - it's ok. 77112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 77212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; " 77312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "prev->hw_next=0x%p\n", __func__, prev_desc, 77412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin next_desc, prev_desc ? prev_desc->hw_next : 0); 77512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 77612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 77712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 77812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin local_irq_save(flags); 77912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 78012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do s/w chaining both for DMA and XOR descriptors */ 78112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prev_desc->hw_next = next_desc; 78212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 78312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 78412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 78512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 78612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 78712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 78812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* bind descriptor to the chain */ 78912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (tail->hw_next) 79012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tail = tail->hw_next; 79112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_linked = tail; 79212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 79312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (prev_desc == xor_last_submit) 79412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do not link to the last submitted CB */ 79512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 79612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_xor_set_link(prev_desc, next_desc); 79712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 79812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 79912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 80012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin local_irq_restore(flags); 80112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 80212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 80312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 80412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_get_src_addr - extract the source address from the descriptor 80512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 80612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 ppc440spe_desc_get_src_addr(struct ppc440spe_adma_desc_slot *desc, 80712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, int src_idx) 80812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 80912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 81012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 81112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 81212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 81312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 81412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 81512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 81612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* May have 0, 1, 2, or 3 sources */ 81712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (dma_hw_desc->opc) { 81812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_NO_OP: 81912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DFILL128: 82012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 82112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DCHECK128: 82212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(src_idx)) { 82312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: try to get %d source for" 82412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " DCHECK128\n", __func__, src_idx); 82512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 82612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 82712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu(dma_hw_desc->sg1l); 82812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MULTICAST: 82912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MV_SG1_SG2: 83012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(src_idx > 2)) { 83112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: try to get %d source from" 83212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " DMA descr\n", __func__, src_idx); 83312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 83412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 83512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_idx) { 83612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (le32_to_cpu(dma_hw_desc->sg1u) & 83712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_WIN_MSK) { 83812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 region; 83912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 84012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_idx == 1) 84112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu( 84212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1l) + 84312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->unmap_len; 84412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 84512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin region = (le32_to_cpu( 84612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1u)) >> 84712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 84812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 84912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin region &= DMA_CUED_REGION_MSK; 85012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (region) { 85112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_RXOR123: 85212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu( 85312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1l) + 85412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (desc->unmap_len << 1); 85512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_RXOR124: 85612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu( 85712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1l) + 85812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (desc->unmap_len * 3); 85912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_RXOR125: 86012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu( 86112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1l) + 86212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (desc->unmap_len << 2); 86312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 86412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR 86512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "%s: try to" 86612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " get src3 for region %02x" 86712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "PPC440SPE_DESC_RXOR12?\n", 86812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, region); 86912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 87012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 87112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 87212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR 87312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "%s: try to get %d" 87412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " source for non-cued descr\n", 87512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, src_idx); 87612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 87712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 87812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 87912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu(dma_hw_desc->sg1l); 88012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 88112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: unknown OPC 0x%02x\n", 88212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dma_hw_desc->opc); 88312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 88412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 88512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu(dma_hw_desc->sg1l); 88612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 88712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* May have up to 16 sources */ 88812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 88912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return xor_hw_desc->ops[src_idx].l; 89012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 89112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 89212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 89312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 89412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 89512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_get_dest_addr - extract the destination address from the 89612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor 89712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 89812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 ppc440spe_desc_get_dest_addr(struct ppc440spe_adma_desc_slot *desc, 89912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, int idx) 90012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 90112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 90212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 90312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 90412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 90512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 90612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 90712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 90812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 90912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(!idx)) 91012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu(dma_hw_desc->sg2l); 91112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu(dma_hw_desc->sg3l); 91212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 91312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 91412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return xor_hw_desc->cbtal; 91512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 91612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 91712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 91812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 91912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 92012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_get_src_num - extract the number of source addresses from 92112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the descriptor 92212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 92312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 ppc440spe_desc_get_src_num(struct ppc440spe_adma_desc_slot *desc, 92412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 92512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 92612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 92712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 92812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 92912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 93012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 93112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 93212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 93312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 93412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (dma_hw_desc->opc) { 93512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_NO_OP: 93612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DFILL128: 93712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 93812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DCHECK128: 93912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 94012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MV_SG1_SG2: 94112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MULTICAST: 94212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 94312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Only for RXOR operations we have more than 94412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * one source 94512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 94612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (le32_to_cpu(dma_hw_desc->sg1u) & 94712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_WIN_MSK) { 94812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR op, there are 2 or 3 sources */ 94912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (((le32_to_cpu(dma_hw_desc->sg1u) >> 95012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF) & 95112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_MSK) == DMA_RXOR12) { 95212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR 1-2 */ 95312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 2; 95412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 95512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR 1-2-3/1-2-4/1-2-5 */ 95612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 3; 95712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 95812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 95912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 96012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 96112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: unknown OPC 0x%02x\n", 96212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dma_hw_desc->opc); 96312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 96412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 96512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 96612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* up to 16 sources */ 96712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 96812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return xor_hw_desc->cbc & XOR_CDCR_OAC_MSK; 96912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 97012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 97112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 97212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 97312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 97412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 97512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 97612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_get_dst_num - get the number of destination addresses in 97712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * this descriptor 97812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 97912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 ppc440spe_desc_get_dst_num(struct ppc440spe_adma_desc_slot *desc, 98012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 98112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 98212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 98312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 98412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 98512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 98612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 98712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* May be 1 or 2 destinations */ 98812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 98912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (dma_hw_desc->opc) { 99012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_NO_OP: 99112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DCHECK128: 99212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 99312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MV_SG1_SG2: 99412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DFILL128: 99512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 99612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MULTICAST: 99712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (desc->dst_cnt == 2) 99812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 2; 99912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 100012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 100112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 100212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: unknown OPC 0x%02x\n", 100312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dma_hw_desc->opc); 100412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 100512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 100612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 100712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Always only 1 destination */ 100812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 100912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 101012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 101112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 101212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 101312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 101412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 101512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 101612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_get_link - get the address of the descriptor that 101712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * follows this one 101812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 101912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc, 102012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 102112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 102212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!desc->hw_next) 102312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 102412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 102512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return desc->hw_next->phys; 102612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 102712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 102812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 102912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_is_aligned - check alignment 103012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 103112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline int ppc440spe_desc_is_aligned( 103212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, int num_slots) 103312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 103412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return (desc->idx & (num_slots - 1)) ? 0 : 1; 103512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 103612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 103712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 103812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_xor_slot_count - get the number of slots necessary for 103912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * XOR operation 104012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 104112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt, 104212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int *slots_per_op) 104312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 104412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 104512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 104612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* each XOR descriptor provides up to 16 source operands */ 104712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS; 104812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 104912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT)) 105012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return slot_cnt; 105112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 105212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: len %d > max %d !!\n", 105312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 105412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 105512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return slot_cnt; 105612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 105712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 105812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 105912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for 106012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * DMA2 PQ operation 106112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 106212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs, 106312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, size_t len) 106412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 106512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin signed long long order = 0; 106612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int state = 0; 106712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int addr_count = 0; 106812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 106912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 1; i < src_cnt; i++) { 107012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t cur_addr = srcs[i]; 107112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t old_addr = srcs[i-1]; 107212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (state) { 107312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 0: 107412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cur_addr == old_addr + len) { 107512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct RXOR */ 107612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 1; 107712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 1; 107812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 107912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 108012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (old_addr == cur_addr + len) { 108112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse RXOR */ 108212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = -1; 108312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 1; 108412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 108512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 108612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 108712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 3; 108812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 108912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 109012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 1: 109112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-2 || (order == -1 109212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin && cur_addr != old_addr - len)) { 109312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 109412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 109512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 109612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cur_addr == old_addr + len*order) { 109712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 2; 109812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 109912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 110012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cur_addr == old_addr + 2*len) { 110112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 2; 110212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 110312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 110412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cur_addr == old_addr + 3*len) { 110512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 2; 110612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 110712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 110812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 110912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 111012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 111112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 111212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 111312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 111412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 2: 111512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 111612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 111712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 111812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 111912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 112012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (state == 3) 112112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 112212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 112312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt <= 1 || (state != 1 && state != 2)) { 112412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n", 112512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, src_cnt, state, addr_count, order); 112612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 112712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("\t[%d] 0x%llx \n", i, srcs[i]); 112812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 112912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 113012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 113112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS; 113212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 113312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 113412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 113512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/****************************************************************************** 113612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ADMA channel low-level routines 113712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ******************************************************************************/ 113812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 113912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 114012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan); 114112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan); 114212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 114312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 114412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine 114512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 114612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_device_clear_eot_status( 114712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 114812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 114912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_regs *dma_reg; 115012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 115112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 *p = chan->device->dma_desc_pool_virt; 115212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *cdb; 115312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 rv, i; 115412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 115512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 115612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 115712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 115812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* read FIFO to ack */ 115912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_reg = chan->device->dma_reg; 116012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while ((rv = ioread32(&dma_reg->csfpl))) { 116112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i = rv & DMA_CDB_ADDR_MSK; 116212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb = (struct dma_cdb *)&p[i - 116312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (u32)chan->device->dma_desc_pool]; 116412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 116512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Clear opcode to ack. This is necessary for 116612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ZeroSum operations only 116712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 116812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb->opc = 0; 116912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 117012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_RXOR_RUN, 117112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &ppc440spe_rxor_state)) { 117212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* probably this is a completed RXOR op, 117312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * get pointer to CDB using the fact that 117412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * physical and virtual addresses of CDB 117512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * in pools have the same offsets 117612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 117712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (le32_to_cpu(cdb->sg1u) & 117812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE) { 117912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is a RXOR */ 118012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_RXOR_RUN, 118112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &ppc440spe_rxor_state); 118212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 118312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 118412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 118512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rv & DMA_CDB_STATUS_MSK) { 118612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* ZeroSum check failed 118712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 118812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 118912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t phys = rv & ~DMA_CDB_MSK; 119012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 119112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 119212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Update the status of corresponding 119312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor. 119412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 119512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &chan->chain, 119612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 119712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->phys == phys) 119812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 119912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 120012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 120112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * if cannot find the corresponding 120212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * slot it's a bug 120312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 120412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(&iter->chain_node == &chan->chain); 120512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 120612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->xor_check_result) { 120712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_PCHECK, 120812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags)) { 120912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *iter->xor_check_result |= 121012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin SUM_CHECK_P_RESULT; 121112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 121212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_QCHECK, 121312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags)) { 121412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *iter->xor_check_result |= 121512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin SUM_CHECK_Q_RESULT; 121612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 121712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 121812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 121912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 122012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 122112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 122212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rv = ioread32(&dma_reg->dsts); 122312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rv) { 122412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("DMA%d err status: 0x%x\n", 122512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id, rv); 122612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* write back to clear */ 122712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(rv, &dma_reg->dsts); 122812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 122912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 123012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 123112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reset status bits to ack */ 123212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 123312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rv = ioread32be(&xor_reg->sr); 123412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(rv, &xor_reg->sr); 123512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 123612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) { 123712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rv & XOR_IE_RPTIE_BIT) { 123812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Read PLB Timeout Error. 123912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Try to resubmit the CB 124012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 124112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 val = ioread32be(&xor_reg->ccbalr); 124212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 124312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(val, &xor_reg->cblalr); 124412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 124512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin val = ioread32be(&xor_reg->crsr); 124612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(val | XOR_CRSR_XAE_BIT, 124712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &xor_reg->crsr); 124812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 124912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("XOR ERR 0x%x status\n", rv); 125012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 125112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 125212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 125312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* if the XORcore is idle, but there are unprocessed CBs 125412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * then refetch the s/w chain here 125512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 125612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) && 125712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin do_xor_refetch) 125812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_append(chan); 125912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 126012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 126112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 126212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 126312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 126412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_is_busy - get the channel status 126512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 126612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan) 126712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 126812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_regs *dma_reg; 126912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 127012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int busy = 0; 127112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 127212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 127312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 127412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 127512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_reg = chan->device->dma_reg; 127612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* if command FIFO's head and tail pointers are equal and 127712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * status tail is the same as command, then channel is free 127812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 127912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) || 128012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp)) 128112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin busy = 1; 128212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 128312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 128412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* use the special status bit for the XORcore 128512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 128612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 128712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0; 128812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 128912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 129012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 129112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return busy; 129212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 129312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 129412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 129512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_set_first_xor_descriptor - init XORcore chain 129612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 129712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_set_first_xor_descriptor( 129812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 129912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *next_desc) 130012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 130112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg = chan->device->xor_reg; 130212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 130312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) 130412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_INFO "%s: Warn: XORcore is running " 130512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "when try to set the first CDB!\n", 130612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 130712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 130812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_submit = xor_last_linked = next_desc; 130912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 131012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr); 131112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 131212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(next_desc->phys, &xor_reg->cblalr); 131312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(0, &xor_reg->cblahr); 131412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT, 131512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &xor_reg->cbcr); 131612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 131712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->hw_chain_inited = 1; 131812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 131912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 132012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 132112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO. 132212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * called with irqs disabled 132312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 132412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan, 132512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc) 132612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 132712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 pcdb; 132812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_regs *dma_reg = chan->device->dma_reg; 132912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 133012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pcdb = desc->phys; 133112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_INT, &desc->flags)) 133212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pcdb |= DMA_CDB_NO_INT; 133312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 133412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan_last_sub[chan->device->id] = desc; 133512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 133612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(print_cb(chan, desc->hw_desc)); 133712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 133812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(pcdb, &dma_reg->cpfpl); 133912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 134012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 134112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 134212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_append - update the h/w chain in the channel 134312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 134412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan) 134512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 134612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 134712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 134812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb; 134912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 cur_desc; 135012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long flags; 135112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 135212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin local_irq_save(flags); 135312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 135412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 135512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 135612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 135712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cur_desc = ppc440spe_chan_get_current_descriptor(chan); 135812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 135912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(cur_desc)) { 136012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = chan_last_sub[chan->device->id]; 136112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!iter); 136212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 136312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* first peer */ 136412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = chan_first_cdb[chan->device->id]; 136512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!iter); 136612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_dma_put_desc(chan, iter); 136712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->hw_chain_inited = 1; 136812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 136912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 137012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* is there something new to append */ 137112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!iter->hw_next) 137212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 137312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 137412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* flush descriptors from the s/w queue to fifo */ 137512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_continue(iter, &chan->chain, chain_node) { 137612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_dma_put_desc(chan, iter); 137712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!iter->hw_next) 137812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 137912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 138012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 138112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 138212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* update h/w links and refetch */ 138312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!xor_last_submit->hw_next) 138412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 138512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 138612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 138712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* the last linked CDB has to generate an interrupt 138812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * that we'd be able to append the next lists to h/w 138912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * regardless of the XOR engine state at the moment of 139012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * appending of these next lists 139112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 139212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb = xor_last_linked->hw_desc; 139312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->cbc |= XOR_CBCR_CBCE_BIT; 139412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 139512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) { 139612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* XORcore is idle. Refetch now */ 139712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin do_xor_refetch = 0; 139812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_xor_set_link(xor_last_submit, 139912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_submit->hw_next); 140012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 140112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(print_cb_list(chan, 140212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_submit->hw_next)); 140312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 140412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_submit = xor_last_linked; 140512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(ioread32be(&xor_reg->crsr) | 140612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT, 140712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &xor_reg->crsr); 140812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 140912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* XORcore is running. Refetch later in the handler */ 141012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin do_xor_refetch = 1; 141112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 141212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 141312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 141412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 141512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 141612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin local_irq_restore(flags); 141712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 141812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 141912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 142012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor 142112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 142212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 142312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan) 142412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 142512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_regs *dma_reg; 142612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 142712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 142812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!chan->hw_chain_inited)) 142912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* h/w descriptor chain is not initialized yet */ 143012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 143112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 143212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 143312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 143412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 143512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_reg = chan->device->dma_reg; 143612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK); 143712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 143812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 143912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ioread32be(&xor_reg->ccbalr); 144012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 144112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 144212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 144312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 144412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 144512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_run - enable the channel 144612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 144712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan) 144812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 144912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 145012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 145112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 145212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 145312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 145412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMAs are always enabled, do nothing */ 145512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 145612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 145712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* drain write buffer */ 145812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 145912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 146012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* fetch descriptor pointed to in <link> */ 146112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT, 146212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &xor_reg->crsr); 146312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 146412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 146512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 146612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 146712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/****************************************************************************** 146812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ADMA device level 146912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ******************************************************************************/ 147012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 147112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan); 147212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan); 147312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 147412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic dma_cookie_t 147512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx); 147612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 147712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx, 147812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index); 147912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 148012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx, 148112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index); 148212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 148312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 148412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx, 148512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *paddr, unsigned long flags); 148612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 148712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx, 148812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index); 148912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 149012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx, 149112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned char mult, int index, int dst_pos); 149212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 149312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx, 149412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t paddr, dma_addr_t qaddr); 149512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 149612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct page *ppc440spe_rxor_srcs[32]; 149712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 149812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 149912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_can_rxor - check if the operands may be processed with RXOR 150012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 150112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len) 150212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 150312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i, order = 0, state = 0; 150412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int idx = 0; 150512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 150612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!(src_cnt > 1))) 150712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 150812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 150912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs)); 151012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 151112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Skip holes in the source list before checking */ 151212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) { 151312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!srcs[i]) 151412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 151512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_srcs[idx++] = srcs[i]; 151612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 151712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt = idx; 151812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 151912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 1; i < src_cnt; i++) { 152012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin char *cur_addr = page_address(ppc440spe_rxor_srcs[i]); 152112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]); 152212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 152312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (state) { 152412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 0: 152512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cur_addr == old_addr + len) { 152612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct RXOR */ 152712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 1; 152812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 1; 152912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (old_addr == cur_addr + len) { 153012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse RXOR */ 153112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = -1; 153212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 1; 153312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 153412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out; 153512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 153612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 1: 153712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((i == src_cnt - 2) || 153812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (order == -1 && cur_addr != old_addr - len)) { 153912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 154012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 154112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if ((cur_addr == old_addr + len * order) || 154212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (cur_addr == old_addr + 2 * len) || 154312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (cur_addr == old_addr + 3 * len)) { 154412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 2; 154512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 154612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 154712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 154812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 154912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 155012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 2: 155112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 155212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 155312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 155412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 155512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 155612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 155712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout: 155812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (state == 1 || state == 2) 155912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 156012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 156112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 156212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 156312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 156412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 156512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_device_estimate - estimate the efficiency of processing 156612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the operation given on this channel. It's assumed that 'chan' is 156712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * capable to process 'cap' type of operation. 156812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @chan: channel to use 156912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @cap: type of transaction 157012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @dst_lst: array of destination pointers 157112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @dst_cnt: number of destination operands 157212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @src_lst: array of source pointers 157312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @src_cnt: number of source operands 157412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @src_sz: size of each source operand 157512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 157612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_estimate(struct dma_chan *chan, 157712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt, 157812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct page **src_lst, int src_cnt, size_t src_sz) 157912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 158012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int ef = 1; 158112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 158212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cap == DMA_PQ || cap == DMA_PQ_VAL) { 158312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* If RAID-6 capabilities were not activated don't try 158412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * to use them 158512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 158612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!ppc440spe_r6_enabled)) 158712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -1; 158812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 158912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* In the current implementation of ppc440spe ADMA driver it 159012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * makes sense to pick out only pq case, because it may be 159112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * processed: 159212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (1) either using Biskup method on DMA2; 159312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (2) or on DMA0/1. 159412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Thus we give a favour to (1) if the sources are suitable; 159512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * else let it be processed on one of the DMA0/1 engines. 159612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * In the sum_product case where destination is also the 159712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * source process it on DMA0/1 only. 159812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 159912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) { 160012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 160112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1]) 160212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ef = 0; /* sum_product case, process on DMA0/1 */ 160312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz)) 160412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ef = 3; /* override (DMA0/1 + idle) */ 160512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 160612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ef = 0; /* can't process on DMA2 if !rxor */ 160712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 160812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 160912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* channel idleness increases the priority */ 161012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(ef) && 161112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan))) 161212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ef++; 161312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 161412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ef; 161512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 161612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 161712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstruct dma_chan * 161812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap, 161912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct page **dst_lst, int dst_cnt, struct page **src_lst, 162012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, size_t src_sz) 162112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 162212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *best_chan = NULL; 162312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc_dma_chan_ref *ref; 162412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int best_rank = -1; 162512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 162612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!src_sz)) 162712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 162812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_sz > PAGE_SIZE) { 162912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 163012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * should a user of the api ever pass > PAGE_SIZE requests 163112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * we sort out cases where temporary page-sized buffers 163212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * are used. 163312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 163412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (cap) { 163512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_PQ: 163612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt == 1 && dst_lst[1] == src_lst[0]) 163712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 163812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt == 2 && dst_lst[1] == src_lst[1]) 163912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 164012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 164112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_PQ_VAL: 164212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_XOR_VAL: 164312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 164412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 164512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 164612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 164712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 164812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 164912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) { 165012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(cap, ref->chan->device->cap_mask)) { 165112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int rank; 165212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 165312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst, 165412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst_cnt, src_lst, src_cnt, src_sz); 165512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rank > best_rank) { 165612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin best_rank = rank; 165712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin best_chan = ref->chan; 165812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 165912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 166012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 166112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 166212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return best_chan; 166312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 166412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinEXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel); 166512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 166612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 166712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_get_group_entry - get group entry with index idx 166812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @tdesc: is the last allocated slot in the group. 166912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 167012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot * 167112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx) 167212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 167312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter = tdesc->group_head; 167412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i = 0; 167512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 167612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) { 167712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n", 167812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt); 167912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 168012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 168112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 168212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &tdesc->group_list, chain_node) { 168312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i++ == entry_idx) 168412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 168512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 168612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return iter; 168712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 168812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 168912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 169012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_free_slots - flags descriptor slots for reuse 169112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @slot: Slot to free 169212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Caller must hold &ppc440spe_chan->lock while calling this function 169312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 169412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot, 169512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 169612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 169712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int stride = slot->slots_per_op; 169812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 169912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (stride--) { 170012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->slots_per_op = 0; 170112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot = list_entry(slot->slot_node.next, 170212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 170312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node); 170412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 170512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 170612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 170712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_unmap(struct ppc440spe_adma_chan *chan, 170812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc) 170912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 171012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 src_cnt, dst_cnt; 171112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr; 171212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 171312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 171412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * get the number of sources & destination 171512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * included in this descriptor and unmap 171612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * them all 171712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 171812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt = ppc440spe_desc_get_src_num(desc, chan); 171912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst_cnt = ppc440spe_desc_get_dst_num(desc, chan); 172012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 172112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* unmap destinations */ 172212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(desc->async_tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) { 172312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (dst_cnt--) { 172412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr = ppc440spe_desc_get_dest_addr( 172512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, chan, dst_cnt); 172612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(chan->device->dev, 172712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr, desc->unmap_len, 172812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_FROM_DEVICE); 172912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 173012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 173112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 173212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* unmap sources */ 173312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(desc->async_tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) { 173412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (src_cnt--) { 173512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr = ppc440spe_desc_get_src_addr( 173612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, chan, src_cnt); 173712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(chan->device->dev, 173812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr, desc->unmap_len, 173912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_TO_DEVICE); 174012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 174112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 174212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 174312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 174412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 174512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_run_tx_complete_actions - call functions to be called 174612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * upon completion 174712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 174812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic dma_cookie_t ppc440spe_adma_run_tx_complete_actions( 174912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 175012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 175112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t cookie) 175212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 175312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 175412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 175512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(desc->async_tx.cookie < 0); 175612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (desc->async_tx.cookie > 0) { 175712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = desc->async_tx.cookie; 175812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->async_tx.cookie = 0; 175912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 176012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* call the callback (must not sleep or submit new 176112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * operations to this channel) 176212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 176312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (desc->async_tx.callback) 176412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->async_tx.callback( 176512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->async_tx.callback_param); 176612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 176712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* unmap dma addresses 176812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (unmap_single vs unmap_page?) 176912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 177012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * actually, ppc's dma_unmap_page() functions are empty, so 177112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the following code is just for the sake of completeness 177212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 177312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan && chan->needs_unmap && desc->group_head && 177412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->unmap_len) { 177512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *unmap = 177612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->group_head; 177712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* assume 1 slot per op always */ 177812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 slot_count = unmap->slot_cnt; 177912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 178012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Run through the group list and unmap addresses */ 178112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < slot_count; i++) { 178212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!unmap); 178312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_unmap(chan, unmap); 178412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unmap = unmap->hw_next; 178512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 178612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 178712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 178812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 178912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* run dependent operations */ 179012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_run_dependencies(&desc->async_tx); 179112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 179212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return cookie; 179312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 179412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 179512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 179612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set) 179712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 179812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc, 179912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 180012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 180112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* the client is allowed to attach dependent operations 180212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * until 'ack' is set 180312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 180412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!async_tx_test_ack(&desc->async_tx)) 180512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 180612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 180712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* leave the last descriptor in the chain 180812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * so we can append to it 180912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 181012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (list_is_last(&desc->chain_node, &chan->chain) || 181112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->phys == ppc440spe_chan_get_current_descriptor(chan)) 181212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 181312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 181412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan->device->id != PPC440SPE_XOR_ID) { 181512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* our DMA interrupt handler clears opc field of 181612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * each processed descriptor. For all types of 181712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * operations except for ZeroSum we do not actually 181812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * need ack from the interrupt handler. ZeroSum is a 181912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * special case since the result of this operation 182012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * is available from the handler only, so if we see 182112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * such type of descriptor (which is unprocessed yet) 182212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * then leave it in chain. 182312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 182412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *cdb = desc->hw_desc; 182512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cdb->opc == DMA_CDB_OPC_DCHECK128) 182612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 182712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 182812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 182912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n", 183012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->phys, desc->idx, desc->slots_per_op); 183112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 183212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&desc->chain_node); 183312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_free_slots(desc, chan); 183412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 183512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 183612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 183712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 183812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine 183912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * which runs through the channel CDBs list until reach the descriptor 184012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * currently processed. When routine determines that all CDBs of group 184112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * are completed then corresponding callbacks (if any) are called and slots 184212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * are freed. 184312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 184412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan) 184512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 184612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL; 184712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t cookie = 0; 184812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 current_desc = ppc440spe_chan_get_current_descriptor(chan); 184912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int busy = ppc440spe_chan_is_busy(chan); 185012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int seen_current = 0, slot_cnt = 0, slots_per_op = 0; 185112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 185212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n", 185312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id, __func__); 185412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 185512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!current_desc) { 185612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* There were no transactions yet, so 185712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * nothing to clean 185812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 185912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return; 186012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 186112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 186212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* free completed slots from the chain starting with 186312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the oldest descriptor 186412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 186512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(iter, _iter, &chan->chain, 186612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 186712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d " 186812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "busy: %d this_desc: %#llx next_desc: %#x " 186912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "cur: %#x ack: %d\n", 187012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->async_tx.cookie, iter->idx, busy, iter->phys, 187112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_get_link(iter, chan), current_desc, 187212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin async_tx_test_ack(&iter->async_tx)); 187312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prefetch(_iter); 187412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prefetch(&_iter->async_tx); 187512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 187612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do not advance past the current descriptor loaded into the 187712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * hardware channel,subsequent descriptors are either in process 187812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * or have not been submitted 187912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 188012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (seen_current) 188112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 188212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 188312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* stop the search if we reach the current descriptor and the 188412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * channel is busy, or if it appears that the current descriptor 188512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * needs to be re-read (i.e. has been appended to) 188612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 188712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->phys == current_desc) { 188812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(seen_current++); 188912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (busy || ppc440spe_desc_get_link(iter, chan)) { 189012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* not all descriptors of the group have 189112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * been completed; exit. 189212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 189312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 189412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 189512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 189612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 189712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* detect the start of a group transaction */ 189812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!slot_cnt && !slots_per_op) { 189912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = iter->slot_cnt; 190012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = iter->slots_per_op; 190112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slot_cnt <= slots_per_op) { 190212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = 0; 190312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = 0; 190412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 190512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 190612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 190712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slot_cnt) { 190812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!group_start) 190912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = iter; 191012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt -= slots_per_op; 191112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 191212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 191312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* all the members of a group are complete */ 191412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slots_per_op != 0 && slot_cnt == 0) { 191512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter; 191612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int end_of_chain = 0; 191712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 191812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* clean up the group */ 191912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = group_start->slot_cnt; 192012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin grp_iter = group_start; 192112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe_from(grp_iter, _grp_iter, 192212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &chan->chain, chain_node) { 192312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 192412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = ppc440spe_adma_run_tx_complete_actions( 192512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin grp_iter, chan, cookie); 192612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 192712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt -= slots_per_op; 192812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin end_of_chain = ppc440spe_adma_clean_slot( 192912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin grp_iter, chan); 193012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (end_of_chain && slot_cnt) { 193112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Should wait for ZeroSum completion */ 193212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cookie > 0) 193312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->completed_cookie = cookie; 193412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return; 193512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 193612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 193712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slot_cnt == 0 || end_of_chain) 193812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 193912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 194012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 194112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* the group should be complete at this point */ 194212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(slot_cnt); 194312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 194412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = 0; 194512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = NULL; 194612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (end_of_chain) 194712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 194812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 194912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 195012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (slots_per_op) /* wait for group completion */ 195112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 195212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 195312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan, 195412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie); 195512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 195612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_adma_clean_slot(iter, chan)) 195712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 195812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 195912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 196012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!seen_current); 196112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 196212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cookie > 0) { 196312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->completed_cookie = cookie; 196412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\tcompleted cookie %d\n", cookie); 196512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 196612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 196712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 196812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 196912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 197012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_tasklet - clean up watch-dog initiator 197112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 197212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_tasklet(unsigned long data) 197312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 197412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan = (struct ppc440spe_adma_chan *) data; 197512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 197612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING); 197712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __ppc440spe_adma_slot_cleanup(chan); 197812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock(&chan->lock); 197912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 198012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 198112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 198212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_slot_cleanup - clean up scheduled initiator 198312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 198412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan) 198512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 198612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&chan->lock); 198712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __ppc440spe_adma_slot_cleanup(chan); 198812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 198912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 199012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 199112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 199212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_alloc_slots - allocate free slots (if any) 199312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 199412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots( 199512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, int num_slots, 199612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slots_per_op) 199712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 199812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter = NULL, *_iter; 199912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *alloc_start = NULL; 200012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct list_head chain = LIST_HEAD_INIT(chain); 200112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slots_found, retry = 0; 200212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 200312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 200412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!num_slots || !slots_per_op); 200512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* start search from the last allocated descrtiptor 200612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * if a contiguous allocation can not be found start searching 200712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * from the beginning of the list 200812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 200912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinretry: 201012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_found = 0; 201112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (retry == 0) 201212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = chan->last_used; 201312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 201412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(&chan->all_slots, 201512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 201612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node); 201712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots, 201812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node) { 201912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prefetch(_iter); 202012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prefetch(&_iter->async_tx); 202112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->slots_per_op) { 202212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_found = 0; 202312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 202412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 202512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 202612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* start the allocation if the slot is correctly aligned */ 202712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!slots_found++) 202812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin alloc_start = iter; 202912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 203012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slots_found == num_slots) { 203112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *alloc_tail = NULL; 203212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *last_used = NULL; 203312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 203412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = alloc_start; 203512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (num_slots) { 203612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 203712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* pre-ack all but the last descriptor */ 203812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (num_slots != slots_per_op) 203912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin async_tx_ack(&iter->async_tx); 204012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 204112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_add_tail(&iter->chain_node, &chain); 204212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin alloc_tail = iter; 204312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->async_tx.cookie = 0; 204412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 204512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->flags = 0; 204612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->slot_cnt = num_slots; 204712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->xor_check_result = NULL; 204812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < slots_per_op; i++) { 204912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->slots_per_op = slots_per_op - i; 205012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin last_used = iter; 205112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->slot_node.next, 205212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 205312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node); 205412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 205512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin num_slots -= slots_per_op; 205612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 205712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin alloc_tail->group_head = alloc_start; 205812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin alloc_tail->async_tx.cookie = -EBUSY; 205912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_splice(&chain, &alloc_tail->group_list); 206012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->last_used = last_used; 206112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return alloc_tail; 206212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 206312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 206412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!retry++) 206512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto retry; 206612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 206712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* try to free some slots if the allocation fails */ 206812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_schedule(&chan->irq_tasklet); 206912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 207012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 207112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 207212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 207312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_alloc_chan_resources - allocate pools for CDB slots 207412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 207512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan) 207612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 207712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 207812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *slot = NULL; 207912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin char *hw_desc; 208012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i, db_sz; 208112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int init; 208212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 208312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 208412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin init = ppc440spe_chan->slots_allocated ? 0 : 1; 208512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->chan_id = ppc440spe_chan->device->id; 208612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 208712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Allocate descriptor slots */ 208812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i = ppc440spe_chan->slots_allocated; 208912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID) 209012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin db_sz = sizeof(struct dma_cdb); 209112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 209212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin db_sz = sizeof(struct xor_cb); 209312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 209412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) { 209512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot), 209612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin GFP_KERNEL); 209712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!slot) { 209812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_INFO "SPE ADMA Channel only initialized" 209912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " %d descriptor slots", i--); 210012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 210112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 210212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 210312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt; 210412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->hw_desc = (void *) &hw_desc[i * db_sz]; 210512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_async_tx_descriptor_init(&slot->async_tx, chan); 210612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->async_tx.tx_submit = ppc440spe_adma_tx_submit; 210712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&slot->chain_node); 210812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&slot->slot_node); 210912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&slot->group_list); 211012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz; 211112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->idx = i; 211212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 211312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 211412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->slots_allocated++; 211512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots); 211612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 211712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 211812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 211912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i && !ppc440spe_chan->last_used) { 212012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->last_used = 212112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_entry(ppc440spe_chan->all_slots.next, 212212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 212312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node); 212412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 212512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 212612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 212712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: allocated %d descriptor slots\n", 212812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, i); 212912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 213012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* initialize the channel and the chain with a null operation */ 213112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (init) { 213212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (ppc440spe_chan->device->id) { 213312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 213412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 213512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->hw_chain_inited = 0; 213612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Use WXOR for self-testing */ 213712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!ppc440spe_r6_tchan) 213812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_r6_tchan = ppc440spe_chan; 213912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 214012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 214112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_start_null_xor(ppc440spe_chan); 214212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 214312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 214412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 214512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 214612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->needs_unmap = 1; 214712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 214812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 214912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return (i > 0) ? i : -ENOMEM; 215012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 215112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 215212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 215312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_assign_cookie - assign a cookie 215412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 215512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic dma_cookie_t ppc440spe_desc_assign_cookie( 215612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 215712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc) 215812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 215912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t cookie = chan->common.cookie; 216012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 216112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie++; 216212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cookie < 0) 216312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = 1; 216412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->common.cookie = desc->async_tx.cookie = cookie; 216512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return cookie; 216612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 216712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 216812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 216912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_rxor_set_region_data - 217012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 217112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc, 217212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 xor_arg_no, u32 mask) 217312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 217412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 217512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 217612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->ops[xor_arg_no].h |= mask; 217712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 217812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 217912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 218012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_rxor_set_src - 218112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 218212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc, 218312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 xor_arg_no, dma_addr_t addr) 218412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 218512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 218612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 218712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE; 218812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->ops[xor_arg_no].l = addr; 218912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 219012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 219112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 219212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_rxor_set_mult - 219312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 219412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc, 219512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 xor_arg_no, u8 idx, u8 mult) 219612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 219712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 219812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 219912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8); 220012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 220112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 220212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 220312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold 220412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * has been achieved 220512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 220612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan) 220712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 220812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n", 220912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id, chan->pending); 221012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 221112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) { 221212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->pending = 0; 221312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_append(chan); 221412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 221512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 221612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 221712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 221812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_tx_submit - submit new descriptor group to the channel 221912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (it's not necessary that descriptors will be submitted to the h/w 222012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * chains too right now) 222112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 222212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx) 222312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 222412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc; 222512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan); 222612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail; 222712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 222812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slots_per_op; 222912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t cookie; 223012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 223112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = tx_to_ppc440spe_adma_slot(tx); 223212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 223312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 223412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = group_start->slot_cnt; 223512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = group_start->slots_per_op; 223612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 223712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&chan->lock); 223812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 223912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = ppc440spe_desc_assign_cookie(chan, sw_desc); 224012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 224112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(list_empty(&chan->chain))) { 224212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* first peer */ 224312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_splice_init(&sw_desc->group_list, &chan->chain); 224412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan_first_cdb[chan->device->id] = group_start; 224512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 224612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* isn't first peer, bind CDBs to chain */ 224712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin old_chain_tail = list_entry(chan->chain.prev, 224812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 224912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 225012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_splice_init(&sw_desc->group_list, 225112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &old_chain_tail->chain_node); 225212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* fix up the hardware chain */ 225312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_link(chan, old_chain_tail, group_start); 225412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 225512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 225612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* increment the pending count by the number of operations */ 225712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->pending += slot_cnt / slots_per_op; 225812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_check_threshold(chan); 225912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 226012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 226112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, 226212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n", 226312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id, __func__, 226412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.cookie, sw_desc->idx, sw_desc); 226512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 226612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return cookie; 226712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 226812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 226912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 227012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation 227112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 227212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt( 227312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, unsigned long flags) 227412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 227512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 227612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 227712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op; 227812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 227912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 228012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 228112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 228212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id, 228312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 228412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 228512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 228612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = slots_per_op = 1; 228712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 228812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op); 228912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 229012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 229112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan); 229212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start->unmap_len = 0; 229312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 229412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 229512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 229612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 229712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 229812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 229912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 230012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 230112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation 230212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 230312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy( 230412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t dma_dest, 230512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t dma_src, size_t len, unsigned long flags) 230612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 230712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 230812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 230912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op; 231012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 231112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 231212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 231312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!len)) 231412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 231512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 2316427cdf19b97e509e21e5d347e18d8b0b34723dfcColy Li BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT); 231712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 231812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 231912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 232012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 232112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s len: %u int_en %d\n", 232212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, __func__, len, 232312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags & DMA_PREP_INTERRUPT ? 1 : 0); 232412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = slots_per_op = 1; 232512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 232612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op); 232712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 232812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 232912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_memcpy(group_start, flags); 233012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_set_dest(group_start, dma_dest, 0); 233112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0); 233212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len); 233312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->unmap_len = len; 233412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 233512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 233612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 233712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 233812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 233912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 234012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 234112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 234212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_memset - prepare CDB for a MEMSET operation 234312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 234412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memset( 234512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t dma_dest, int value, 234612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size_t len, unsigned long flags) 234712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 234812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 234912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 235012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op; 235112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 235212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 235312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 235412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!len)) 235512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 235612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 2357427cdf19b97e509e21e5d347e18d8b0b34723dfcColy Li BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT); 235812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 235912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 236012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 236112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 236212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s cal: %u len: %u int_en %d\n", 236312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, __func__, value, len, 236412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags & DMA_PREP_INTERRUPT ? 1 : 0); 236512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 236612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = slots_per_op = 1; 236712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 236812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op); 236912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 237012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 237112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_memset(group_start, value, flags); 237212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_set_dest(group_start, dma_dest, 0); 237312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len); 237412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->unmap_len = len; 237512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 237612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 237712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 237812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 237912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 238012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 238112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 238212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 238312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation 238412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 238512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor( 238612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t dma_dest, 238712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dma_src, u32 src_cnt, size_t len, 238812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long flags) 238912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 239012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 239112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 239212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op; 239312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 239412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 239512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 239612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id, 239712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_dest, dma_src, src_cnt)); 239812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!len)) 239912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 2400427cdf19b97e509e21e5d347e18d8b0b34723dfcColy Li BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 240112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 240212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 240312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n", 240412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, __func__, src_cnt, len, 240512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags & DMA_PREP_INTERRUPT ? 1 : 0); 240612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 240712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 240812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op); 240912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 241012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op); 241112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 241212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 241312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_xor(group_start, src_cnt, flags); 241412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_set_dest(group_start, dma_dest, 0); 241512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (src_cnt--) 241612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_memcpy_xor_set_src(group_start, 241712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_src[src_cnt], src_cnt); 241812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len); 241912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->unmap_len = len; 242012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 242112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 242212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 242312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 242412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 242512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 242612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 242712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline void 242812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc, 242912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt); 243012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor); 243112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 243212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 243312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_init_dma2rxor_slot - 243412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 243512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_init_dma2rxor_slot( 243612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 243712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *src, int src_cnt) 243812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 243912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 244012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 244112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* initialize CDB */ 244212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) { 244312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i, 244412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt, (u32)src[i]); 244512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 244612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 244712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 244812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 244912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma01_prep_mult - 245012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for Q operation where destination is also the source 245112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 245212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult( 245312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan, 245412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 245512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf, size_t len, unsigned long flags) 245612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 245712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL; 245812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 245912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 246012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 246112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_WXOR, &op); 246212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = 2; 246312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 246412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 246512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 246612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* use WXOR, each descriptor occupies one slot */ 246712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 246812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 246912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 247012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 247112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 247212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 247312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 247412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bits(op, &sw_desc->flags); 247512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->src_cnt = src_cnt; 247612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt = dst_cnt; 247712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* First descriptor, zero data in the destination and copy it 247812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * to q page using MULTICAST transfer. 247912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 248012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&sw_desc->group_list, 248112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 248212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 248312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 248412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 248512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 248612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 248712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 248812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 248912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 249012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MULTICAST; 249112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 249212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 249312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, dst[0], 0); 249412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1); 249512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 249612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src[0]); 249712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 249812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 249912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 250012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 250112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Second descriptor, multiply data from the q page 250212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * and store the result in real destination. 250312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 250412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 250512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 250612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 250712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 250812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 250912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 251012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 251112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 251212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 251312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 251412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 251512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 251612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 251712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB, dst[1]); 251812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 251912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, dst[0], 0); 252012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 252112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 252212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1, scf[0]); 252312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 252412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 252512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 252612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 252712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 252812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 252912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 253012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc; 253112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 253212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 253312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 253412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma01_prep_sum_product - 253512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also 253612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the source. 253712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 253812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product( 253912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan, 254012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dst, dma_addr_t *src, int src_cnt, 254112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf, size_t len, unsigned long flags) 254212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 254312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL; 254412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 254512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 254612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 254712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_WXOR, &op); 254812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = 3; 254912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 255012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 255112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 255212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR, each descriptor occupies one slot */ 255312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 255412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 255512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 255612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 255712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 255812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 255912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 256012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bits(op, &sw_desc->flags); 256112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->src_cnt = src_cnt; 256212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt = 1; 256312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 1st descriptor, src[1] data to q page and zero destination */ 256412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&sw_desc->group_list, 256512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 256612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 256712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 256812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 256912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 257012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 257112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 257212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 257312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MULTICAST; 257412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 257512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 257612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *dst, 0); 257712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 0, 257812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->qdest, 1); 257912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 258012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src[1]); 258112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 258212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 258312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 258412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 2nd descriptor, multiply src[1] data and store the 258512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * result in destination */ 258612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 258712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 258812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 258912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 259012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 259112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 259212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 259312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 259412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 259512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 259612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 259712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 259812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 259912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 260012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 260112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 260212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->qdest); 260312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 260412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *dst, 0); 260512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 260612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1, scf[1]); 260712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 260812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 260912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 261012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 261112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 3rd descriptor, multiply src[0] data and xor it 261212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * with destination 261312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 261412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 261512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 261612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 261712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 261812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 261912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 262012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 262112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 262212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 262312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 262412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 262512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 262612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 262712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src[0]); 262812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 262912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *dst, 0); 263012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 263112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1, scf[0]); 263212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 263312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 263412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 263512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 263612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 263712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 263812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 263912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc; 264012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 264112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 264212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq( 264312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan, 264412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 264512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf, size_t len, unsigned long flags) 264612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 264712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 264812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter; 264912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 265012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned char mult = 1; 265112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 265212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n", 265312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dst_cnt, src_cnt, len); 265412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* select operations WXOR/RXOR depending on the 265512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * source addresses of operators and the number 265612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * of destinations (RXOR support only Q-parity calculations) 265712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 265812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_WXOR, &op); 265912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) { 266012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* no active RXOR; 266112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * do RXOR if: 266212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - there are more than 1 source, 266312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - len is aligned on 512-byte boundary, 266412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - source addresses fit to one of 4 possible regions. 266512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 266612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt > 1 && 266712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin !(len & MQ0_CF2H_RXOR_BS_MASK) && 266812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (src[0] + len) == src[1]) { 266912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* may do RXOR R1 R2 */ 267012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR, &op); 267112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt != 2) { 267212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* may try to enhance region of RXOR */ 267312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((src[1] + len) == src[2]) { 267412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 R3 */ 267512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR123, 267612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &op); 267712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if ((src[1] + len * 2) == src[2]) { 267812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 R4 */ 267912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR124, &op); 268012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if ((src[1] + len * 3) == src[2]) { 268112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 R5 */ 268212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR125, 268312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &op); 268412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 268512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 */ 268612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR12, 268712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &op); 268812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 268912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 269012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 */ 269112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR12, &op); 269212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 269312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 269412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 269512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_RXOR, &op)) { 269612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* can not do this operation with RXOR */ 269712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_RXOR_RUN, 269812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &ppc440spe_rxor_state); 269912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 270012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* can do; set block size right now */ 270112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_rxor_block_size(len); 270212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 270312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 270412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 270512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Number of necessary slots depends on operation type selected */ 270612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_RXOR, &op)) { 270712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is a WXOR only chain. Need descriptors for each 270812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * source to GF-XOR them with WXOR, and need descriptors 270912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for each destination to zero them with WXOR 271012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 271112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = src_cnt; 271212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 271312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_P) { 271412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt++; 271512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_P, &op); 271612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 271712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_Q) { 271812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt++; 271912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_Q, &op); 272012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 272112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 272212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Need 1/2 descriptor for RXOR operation, and 272312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * need (src_cnt - (2 or 3)) for WXOR of sources 272412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * remained (if any) 272512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 272612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = dst_cnt; 272712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 272812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_P) 272912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_P, &op); 273012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_Q) 273112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_Q, &op); 273212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 273312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_RXOR12, &op)) 273412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt += src_cnt - 2; 273512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 273612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt += src_cnt - 3; 273712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 273812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Thus we have either RXOR only chain or 273912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * mixed RXOR/WXOR 274012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 274112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slot_cnt == dst_cnt) 274212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR only chain */ 274312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_WXOR, &op); 274412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 274512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 274612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 274712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* for both RXOR/WXOR each descriptor occupies one slot */ 274812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 274912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 275012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt, 275112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags, op); 275212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 275312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* setup dst/src/mult */ 275412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n", 275512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dst[0], dst[1]); 275612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_dest(sw_desc, dst, flags); 275712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (src_cnt--) { 275812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt], 275912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt); 276012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 276112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* NOTE: "Multi = 0 is equivalent to = 1" as it 276212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf 276312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * doesn't work for RXOR with DMA0/1! Instead, multi=0 276412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * leads to zeroing source data after RXOR. 276512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * So, for P case set-up mult=1 explicitly. 276612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 276712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 276812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult = scf[src_cnt]; 276912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src_mult(sw_desc, 277012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult, src_cnt, dst_cnt - 1); 277112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 277212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 277312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup byte count foreach slot just allocated */ 277412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 277512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, 277612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 277712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, 277812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan, len); 277912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 278012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 278112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 278212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 278312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 278412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc; 278512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 278612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 278712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq( 278812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan, 278912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 279012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf, size_t len, unsigned long flags) 279112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 279212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, descs_per_op; 279312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter; 279412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 279512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned char mult = 1; 279612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 279712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!dst_cnt); 279812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n", 279912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dst_cnt, src_cnt, len);*/ 280012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 280112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 280212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len); 280312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (descs_per_op < 0) { 280412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 280512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 280612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 280712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 280812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* depending on number of sources we have 1 or 2 RXOR chains */ 280912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = descs_per_op * dst_cnt; 281012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 281112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 281212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 281312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op = slot_cnt; 281412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 281512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 281612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt, 281712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin --op ? 0 : flags); 281812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 281912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len); 282012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 282112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 282212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_init_rxor_cursor(&(iter->rxor_cursor)); 282312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->rxor_cursor.len = len; 282412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->descs_per_op = descs_per_op; 282512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 282612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op = 0; 282712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 282812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op++; 282912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (op % descs_per_op == 0) 283012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_init_dma2rxor_slot(iter, src, 283112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt); 283212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(!list_is_last(&iter->chain_node, 283312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list))) { 283412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 283512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = 283612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_entry(iter->chain_node.next, 283712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 283812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 283912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_xor_set_link(iter, iter->hw_next); 284012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 284112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is the last descriptor. */ 284212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 284312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 284412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 284512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 284612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* fixup head descriptor */ 284712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt = dst_cnt; 284812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_P) 284912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_P, &sw_desc->flags); 285012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_Q) 285112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags); 285212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 285312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* setup dst/src/mult */ 285412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_dest(sw_desc, dst, flags); 285512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 285612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (src_cnt--) { 285712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* handle descriptors (if dst_cnt == 2) inside 285812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the ppc440spe_adma_pq_set_srcxxx() functions 285912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 286012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt], 286112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt); 286212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 286312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult = scf[src_cnt]; 286412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src_mult(sw_desc, 286512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult, src_cnt, dst_cnt - 1); 286612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 286712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 286812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 286912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_rxor_block_size(len); 287012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc; 287112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 287212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 287312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 287412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation 287512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 287612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq( 287712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, 287812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt, const unsigned char *scf, 287912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size_t len, unsigned long flags) 288012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 288112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 288212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL; 288312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int dst_cnt = 0; 288412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 288512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 288612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 288712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id, 288812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst, src, src_cnt)); 288912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!len); 2890427cdf19b97e509e21e5d347e18d8b0b34723dfcColy Li BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 289112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!src_cnt); 289212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 289312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt == 1 && dst[1] == src[0]) { 289412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t dest[2]; 289512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 289612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* dst[1] is real destination (Q) */ 289712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dest[0] = dst[1]; 289812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is the page to multicast source data to */ 289912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dest[1] = ppc440spe_chan->qdest; 290012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan, 290112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dest, 2, src, src_cnt, scf, len, flags); 290212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 290312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 290412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 290512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt == 2 && dst[1] == src[1]) { 290612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan, 290712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &dst[1], src, 2, scf, len, flags); 290812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 290912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 291012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 291112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(flags & DMA_PREP_PQ_DISABLE_P)) { 291212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!dst[0]); 291312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst_cnt++; 291412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags |= DMA_PREP_ZERO_P; 291512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 291612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 291712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(flags & DMA_PREP_PQ_DISABLE_Q)) { 291812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!dst[1]); 291912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst_cnt++; 292012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags |= DMA_PREP_ZERO_Q; 292112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 292212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 292312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!dst_cnt); 292412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 292512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 292612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n", 292712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, __func__, src_cnt, len, 292812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags & DMA_PREP_INTERRUPT ? 1 : 0); 292912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 293012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (ppc440spe_chan->device->id) { 293112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 293212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 293312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan, 293412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst, dst_cnt, src, src_cnt, scf, 293512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len, flags); 293612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 293712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 293812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 293912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan, 294012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst, dst_cnt, src, src_cnt, scf, 294112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len, flags); 294212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 294312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 294412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 294512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 294612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 294712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 294812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 294912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for 295012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * a PQ_ZERO_SUM operation 295112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 295212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum( 295312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, 295412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt, const unsigned char *scf, size_t len, 295512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin enum sum_check_flags *pqres, unsigned long flags) 295612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 295712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 295812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *iter; 295912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t pdest, qdest; 296012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op, idst, dst_cnt; 296112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 296212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 296312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 296412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_PQ_DISABLE_P) 296512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pdest = 0; 296612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 296712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pdest = pq[0]; 296812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 296912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_PQ_DISABLE_Q) 297012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qdest = 0; 297112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 297212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qdest = pq[1]; 297312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 297412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id, 297512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src, src_cnt, scf)); 297612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 297712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Always use WXOR for P/Q calculations (two destinations). 297812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Need 1 or 2 extra slots to verify results are zero. 297912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 298012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idst = dst_cnt = (pdest && qdest) ? 2 : 1; 298112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 298212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* One additional slot per destination to clone P/Q 298312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * before calculation (we have to preserve destinations). 298412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 298512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = src_cnt + dst_cnt * 2; 298612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = 1; 298712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 298812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 298912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 299012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op); 299112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 299212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt); 299312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 299412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup byte count for each slot just allocated */ 299512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 299612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 299712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 299812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len); 299912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 300012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 300112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 300212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (pdest) { 300312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 300412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 300512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 300612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = sw_desc->group_head; 300712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 300812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 300912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 301012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 301112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 301212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 301312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 301412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->src_cnt = 0; 301512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->dst_cnt = 0; 301612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 0, 301712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->pdest, 0); 301812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest); 301912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 302012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len); 302112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = 0; 302212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* override pdest to preserve original P */ 302312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pdest = ppc440spe_chan->pdest; 302412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 302512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (qdest) { 302612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 302712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 302812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 302912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&sw_desc->group_list, 303012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 303112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 303212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 303312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 303412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (pdest) { 303512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 303612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 303712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 303812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 303912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 304012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 304112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 304212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 304312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 304412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 304512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 304612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->src_cnt = 0; 304712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->dst_cnt = 0; 304812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 0, 304912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->qdest, 0); 305012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest); 305112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 305212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len); 305312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = 0; 305412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* override qdest to preserve original Q */ 305512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qdest = ppc440spe_chan->qdest; 305612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 305712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 305812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup destinations for P/Q ops */ 305912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest); 306012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 306112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup zero QWORDs into DCHECK CDBs */ 306212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idst = dst_cnt; 306312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_reverse(iter, &sw_desc->group_list, 306412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 306512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 306612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * The last CDB corresponds to Q-parity check, 306712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the one before last CDB corresponds 306812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * P-parity check 306912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 307012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (idst == DMA_DEST_MAX_NUM) { 307112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (idst == dst_cnt) { 307212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_QCHECK, 307312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags); 307412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 307512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_PCHECK, 307612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags); 307712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 307812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 307912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (qdest) { 308012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_QCHECK, 308112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags); 308212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 308312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_PCHECK, 308412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags); 308512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 308612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 308712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->xor_check_result = pqres; 308812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 308912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 309012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * set it to zero, if check fail then result will 309112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * be updated 309212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 309312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *iter->xor_check_result = 0; 309412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dcheck(iter, ppc440spe_chan, 309512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_qword); 309612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 309712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(--dst_cnt)) 309812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 309912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 310012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 310112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup sources and mults for P/Q ops */ 310212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_continue_reverse(iter, &sw_desc->group_list, 310312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 310412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 310512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 mult_dst; 310612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 310712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 310812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 310912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB, 311012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src[src_cnt - 1]); 311112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (qdest) { 311212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 : 311312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1; 311412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, 311512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_MULT1_OFF, 311612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst, 311712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin scf[src_cnt - 1]); 311812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 311912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(--src_cnt)) 312012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 312112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 312212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 312312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 312412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 312512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 312612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 312712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 312812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for 312912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * XOR ZERO_SUM operation 313012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 313112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum( 313212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt, 313312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size_t len, enum sum_check_flags *result, unsigned long flags) 313412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 313512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_async_tx_descriptor *tx; 313612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t pq[2]; 313712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 313812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* validate P, disable Q */ 313912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pq[0] = src[0]; 314012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pq[1] = 0; 314112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags |= DMA_PREP_PQ_DISABLE_Q; 314212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 314312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1], 314412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt - 1, 0, len, 314512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin result, flags); 314612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return tx; 314712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 314812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 314912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 315012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_set_dest - set destination address into descriptor 315112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 315212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc, 315312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index) 315412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 315512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 315612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 315712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(index >= sw_desc->dst_cnt); 315812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 315912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 316012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 316112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 316212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 316312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 316412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* to do: support transfers lengths > 316512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT 316612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 316712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(sw_desc->group_head, 316812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan, 0, addr, index); 316912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 317012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 317112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_get_group_entry(sw_desc, index); 317212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(sw_desc, 317312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan, 0, addr, index); 317412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 317512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 317612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 317712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 317812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter, 317912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, dma_addr_t addr) 318012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 318112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* To clear destinations update the descriptor 318212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (P or Q depending on index) as follows: 318312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * addr is destination (0 corresponds to SG2): 318412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 318512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0); 318612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 318712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* ... and the addr is source: */ 318812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr); 318912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 319012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* addr is always SG2 then the mult is always DST1 */ 319112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 319212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1, 1); 319312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 319412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 319512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 319612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_pq_set_dest - set destination address into descriptor 319712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for the PQXOR operation 319812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 319912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc, 320012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *addrs, unsigned long flags) 320112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 320212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 320312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 320412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t paddr, qaddr; 320512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr = 0, ppath, qpath; 320612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index = 0, i; 320712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 320812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 320912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 321012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_PQ_DISABLE_P) 321112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr = 0; 321212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 321312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr = addrs[0]; 321412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 321512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_PQ_DISABLE_Q) 321612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qaddr = 0; 321712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 321812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qaddr = addrs[1]; 321912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 322012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!paddr || !qaddr) 322112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr = paddr ? paddr : qaddr; 322212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 322312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 322412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 322512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 322612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* walk through the WXOR source list and set P/Q-destinations 322712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for each slot: 322812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 322912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 323012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is WXOR-only chain; may have 1/2 zero descs */ 323112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 323212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index++; 323312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 323412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index++; 323512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 323612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, index); 323712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (addr) { 323812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* one destination */ 323912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, 324012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list, chain_node) 324112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 324212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, addr, 0); 324312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 324412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* two destinations */ 324512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, 324612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list, chain_node) { 324712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 324812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, paddr, 0); 324912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 325012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, qaddr, 1); 325112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 325212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 325312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 325412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index) { 325512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* To clear destinations update the descriptor 325612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (1st,2nd, or both depending on flags) 325712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 325812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index = 0; 325912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, 326012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) { 326112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry( 326212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc, index++); 326312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_zero_op(iter, chan, 326412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr); 326512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 326612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 326712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, 326812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) { 326912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry( 327012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc, index++); 327112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_zero_op(iter, chan, 327212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qaddr); 327312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 327412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 327512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return; 327612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 327712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 327812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is RXOR-only or RXOR/WXOR mixed chain */ 327912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 328012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* If we want to include destination into calculations, 328112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * then make dest addresses cued with mult=1 (XOR). 328212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 328312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ? 328412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB : 328512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE | 328612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << DMA_CUED_MULT1_OFF); 328712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ? 328812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB : 328912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE | 329012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << DMA_CUED_MULT1_OFF); 329112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 329212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup destination(s) in RXOR slot(s) */ 329312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, index++); 329412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 329512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr ? ppath : qpath, 329612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr ? paddr : qaddr, 0); 329712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addr) { 329812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* two destinations */ 329912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 330012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index++); 330112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 330212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qpath, qaddr, 0); 330312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 330412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 330512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) { 330612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup destination(s) in remaining WXOR 330712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * slots 330812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 330912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 331012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index); 331112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (addr) { 331212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* one destination */ 331312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, 331412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list, 331512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) 331612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr( 331712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter, chan, 331812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, 331912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr, 0); 332012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 332112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 332212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* two destinations */ 332312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, 332412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list, 332512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 332612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr( 332712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter, chan, 332812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, 332912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr, 0); 333012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr( 333112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter, chan, 333212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, 333312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qaddr, 1); 333412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 333512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 333612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 333712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 333812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 333912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 334012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 334112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 334212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA2 descriptors have only 1 destination, so there are 334312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * two chains - one for each dest. 334412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * If we want to include destination into calculations, 334512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * then make dest addresses cued with mult=1 (XOR). 334612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 334712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ? 334812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB : 334912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE | 335012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << DMA_CUED_MULT1_OFF); 335112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 335212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ? 335312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB : 335412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE | 335512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << DMA_CUED_MULT1_OFF); 335612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 335712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 0); 335812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < sw_desc->descs_per_op; i++) { 335912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 336012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr ? ppath : qpath, 336112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr ? paddr : qaddr, 0); 336212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 336312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 336412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 336512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 336612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 336712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addr) { 336812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Two destinations; setup Q here */ 336912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 337012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->descs_per_op); 337112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < sw_desc->descs_per_op; i++) { 337212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, 337312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan, qpath, qaddr, 0); 337412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 337512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 337612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 337712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 337812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 337912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 338012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 338112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 338212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 338312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 338412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 338512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor 338612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for the PQ_ZERO_SUM operation 338712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 338812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pqzero_sum_set_dest( 338912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, 339012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t paddr, dma_addr_t qaddr) 339112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 339212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter, *end; 339312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 339412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr = 0; 339512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int idx; 339612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 339712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 339812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 339912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* walk through the WXOR source list and set P/Q-destinations 340012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for each slot 340112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 340212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idx = (paddr && qaddr) ? 2 : 1; 340312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set end */ 340412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_reverse(end, &sw_desc->group_list, 340512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 340612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(--idx)) 340712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 340812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 340912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set start */ 341012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idx = (paddr && qaddr) ? 2 : 1; 341112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, idx); 341212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 341312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (paddr && qaddr) { 341412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* two destinations */ 341512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &sw_desc->group_list, 341612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 341712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(iter == end)) 341812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 341912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 342012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, paddr, 0); 342112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 342212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, qaddr, 1); 342312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 342412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 342512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* one destination */ 342612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr = paddr ? paddr : qaddr; 342712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &sw_desc->group_list, 342812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 342912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(iter == end)) 343012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 343112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 343212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, addr, 0); 343312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 343412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 343512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 343612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* The remaining descriptors are DATACHECK. These have no need in 343712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * destination. Actually, these destinations are used there 343812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * as sources for check operation. So, set addr as source. 343912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 344012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr); 344112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 344212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addr) { 344312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin end = list_entry(end->chain_node.next, 344412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 344512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr); 344612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 344712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 344812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 344912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 345012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor 345112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 345212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline void ppc440spe_desc_set_xor_src_cnt( 345312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 345412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt) 345512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 345612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *hw_desc = desc->hw_desc; 345712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 345812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc &= ~XOR_CDCR_OAC_MSK; 345912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc |= src_cnt; 346012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 346112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 346212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 346312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_pq_set_src - set source address into descriptor 346412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 346512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc, 346612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index) 346712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 346812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 346912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t haddr = 0; 347012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter = NULL; 347112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 347212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 347312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 347412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 347512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 347612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 347712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain 347812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 347912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 348012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR-only or RXOR/WXOR operation */ 348112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int iskip = test_bit(PPC440SPE_DESC_RXOR12, 348212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags) ? 2 : 3; 348312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 348412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == 0) { 348512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 1st slot (RXOR) */ 348612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* setup sources region (R1-2-3, R1-2-4, 348712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * or R1-2-5) 348812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 348912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_RXOR12, 349012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) 349112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_RXOR12 << 349212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 349312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else if (test_bit(PPC440SPE_DESC_RXOR123, 349412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) 349512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_RXOR123 << 349612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 349712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else if (test_bit(PPC440SPE_DESC_RXOR124, 349812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) 349912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_RXOR124 << 350012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 350112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else if (test_bit(PPC440SPE_DESC_RXOR125, 350212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) 350312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_RXOR125 << 350412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 350512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 350612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 350712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr |= DMA_CUED_XOR_BASE; 350812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 0); 350912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (index < iskip) { 351012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 1st slot (RXOR) 351112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * shall actually set source address only once 351212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * instead of first <iskip> 351312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 351412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = NULL; 351512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 351612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 2nd/3d and next slots (WXOR); 351712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * skip first slot with RXOR 351812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 351912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_CUED_XOR_HB; 352012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 352112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index - iskip + sw_desc->dst_cnt); 352212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 352312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 352412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int znum = 0; 352512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 352612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR-only operation; skip first slots with 352712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * zeroing destinations 352812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 352912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 353012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin znum++; 353112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 353212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin znum++; 353312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 353412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_CUED_XOR_HB; 353512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 353612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index + znum); 353712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 353812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 353912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(iter)) { 354012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr); 354112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 354212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!index && 354312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) && 354412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt == 2) { 354512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* if we have two destinations for RXOR, then 354612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * setup source in the second descr too 354712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 354812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 1); 354912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 355012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr, addr); 355112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 355212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 355312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 355412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 355512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 355612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA2 may do Biskup */ 355712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = sw_desc->group_head; 355812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->dst_cnt == 2) { 355912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* both P & Q calculations required; set P src here */ 356012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_set_src(iter, index, addr); 356112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 356212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is for Q */ 356312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 356412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->descs_per_op); 356512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 356612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_set_src(iter, index, addr); 356712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 356812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 356912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 357012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 357112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 357212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor 357312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 357412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_memcpy_xor_set_src( 357512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, 357612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index) 357712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 357812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 357912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 358012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 358112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = sw_desc->group_head; 358212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 358312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(sw_desc)) 358412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr); 358512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 358612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 358712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 358812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_inc_addr - 358912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 359012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_inc_addr( 359112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 359212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_rxor *cursor, int index, int src_cnt) 359312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 359412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count++; 359512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt - 1) { 359612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count); 359712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cursor->addr_count == XOR_MAX_OPS) { 359812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count); 359912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count = 0; 360012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->desc_count++; 360112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 360212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 360312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 360412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 360512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB 360612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 360712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_dma2rxor_prep_src( 360812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *hdesc, 360912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_rxor *cursor, int index, 361012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, u32 addr) 361112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 361212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int rval = 0; 361312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 sign; 361412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc = hdesc; 361512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 361612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 361712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < cursor->desc_count; i++) { 361812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc = list_entry(hdesc->chain_node.next, 361912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 362012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 362112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 362212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 362312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (cursor->state) { 362412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 0: 362512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (addr == cursor->addrl + cursor->len) { 362612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct RXOR */ 362712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 1; 362812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count++; 362912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 363012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 363112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 363212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR12 << DMA_CUED_REGION_OFF); 363312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 363412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 363512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 363612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cursor->addrl == addr + cursor->len) { 363712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse RXOR */ 363812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 1; 363912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count++; 364012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(cursor->addr_count, &desc->reverse_flags[0]); 364112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 364212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 364312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 364412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR12 << DMA_CUED_REGION_OFF); 364512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 364612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 364712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 364812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 364912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "Cannot build " 365012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "DMA2 RXOR command block.\n"); 365112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 365212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 365312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 365412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 1: 365512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sign = test_bit(cursor->addr_count, 365612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->reverse_flags) 365712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ? -1 : 1; 365812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-2 || (sign == -1 365912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin && addr != cursor->addrl - 2*cursor->len)) { 366012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 0; 366112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 1; 366212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addrl = addr; 366312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 366412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 366512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR12 << DMA_CUED_REGION_OFF); 366612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 366712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 366812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (addr == cursor->addrl + 2*sign*cursor->len) { 366912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 2; 367012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 0; 367112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 367212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 367312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR123 << DMA_CUED_REGION_OFF); 367412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 367512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 367612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 367712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 367812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (addr == cursor->addrl + 3*cursor->len) { 367912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 2; 368012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 0; 368112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 368212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 368312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR124 << DMA_CUED_REGION_OFF); 368412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 368512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 368612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 368712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 368812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (addr == cursor->addrl + 4*cursor->len) { 368912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 2; 369012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 0; 369112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 369212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 369312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR125 << DMA_CUED_REGION_OFF); 369412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 369512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 369612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 369712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 369812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 369912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 0; 370012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 1; 370112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addrl = addr; 370212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 370312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 370412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR12 << DMA_CUED_REGION_OFF); 370512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 370612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 370712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 370812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 370912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 2: 371012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 0; 371112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addrl = addr; 371212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count++; 371312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index) { 371412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 371512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 371612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 371712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 371812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 371912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 372012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return rval; 372112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 372212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 372312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 372412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that 372512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call 372612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 372712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_set_src( 372812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 372912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index, dma_addr_t addr) 373012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 373112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 373212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int k = 0, op = 0, lop = 0; 373312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 373412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* get the RXOR operand which corresponds to index addr */ 373512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (op <= index) { 373612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin lop = op; 373712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (k == XOR_MAX_OPS) { 373812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin k = 0; 373912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc = list_entry(desc->chain_node.next, 374012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 374112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb = desc->hw_desc; 374212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 374312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 374412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) == 374512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (DMA_RXOR12 << DMA_CUED_REGION_OFF)) 374612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op += 2; 374712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 374812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op += 3; 374912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 375012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 375112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(k < 1); 375212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 375312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(k-1, desc->reverse_flags)) { 375412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse operand order; put last op in RXOR group */ 375512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == op - 1) 375612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_src(desc, k - 1, addr); 375712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 375812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct operand order; put first op in RXOR group */ 375912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == lop) 376012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_src(desc, k - 1, addr); 376112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 376212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 376312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 376412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 376512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that 376612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call 376712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 376812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_set_mult( 376912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 377012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index, u8 mult) 377112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 377212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 377312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int k = 0, op = 0, lop = 0; 377412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 377512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* get the RXOR operand which corresponds to index mult */ 377612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (op <= index) { 377712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin lop = op; 377812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (k == XOR_MAX_OPS) { 377912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin k = 0; 378012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc = list_entry(desc->chain_node.next, 378112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 378212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 378312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb = desc->hw_desc; 378412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 378512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 378612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) == 378712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (DMA_RXOR12 << DMA_CUED_REGION_OFF)) 378812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op += 2; 378912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 379012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op += 3; 379112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 379212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 379312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(k < 1); 379412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(k-1, desc->reverse_flags)) { 379512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse order */ 379612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult); 379712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 379812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct order */ 379912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult); 380012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 380112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 380212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 380312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 380412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_init_rxor_cursor - 380512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 380612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor) 380712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 380812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(cursor, 0, sizeof(struct ppc440spe_rxor)); 380912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 2; 381012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 381112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 381212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 381312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into 381412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor for the PQXOR operation 381512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 381612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pq_set_src_mult( 381712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, 381812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned char mult, int index, int dst_pos) 381912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 382012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 382112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 mult_idx, mult_dst; 382212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL; 382312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 382412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 382512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 382612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 382712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 382812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 382912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 383012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int region = test_bit(PPC440SPE_DESC_RXOR12, 383112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags) ? 2 : 3; 383212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 383312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index < region) { 383412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR multipliers */ 383512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 383612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt - 1); 383712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc->dst_cnt == 2) 383812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter1 = ppc440spe_get_group_entry( 383912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc, 0); 384012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 384112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx = DMA_CUED_MULT1_OFF + (index << 3); 384212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst = DMA_CDB_SG_SRC; 384312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 384412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR multiplier */ 384512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 384612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index - region + 384712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt); 384812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx = DMA_CUED_MULT1_OFF; 384912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst = dst_pos ? DMA_CDB_SG_DST2 : 385012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1; 385112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 385212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 385312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int znum = 0; 385412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 385512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR-only; 385612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * skip first slots with destinations (if ZERO_DST has 385712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * place) 385812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 385912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 386012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin znum++; 386112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 386212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin znum++; 386312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 386412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, index + znum); 386512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx = DMA_CUED_MULT1_OFF; 386612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1; 386712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 386812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 386912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(iter)) { 387012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, 387112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx, mult_dst, mult); 387212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 387312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(iter1)) { 387412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* if we have two destinations for RXOR, then 387512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * we've just set Q mult. Set-up P now. 387612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 387712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter1, chan, 387812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx, mult_dst, 1); 387912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 388012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 388112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 388212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 388312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 388412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 388512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = sw_desc->group_head; 388612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc->dst_cnt == 2) { 388712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* both P & Q calculations required; set P mult here */ 388812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_set_mult(iter, index, 1); 388912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 389012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* and then set Q mult */ 389112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 389212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->descs_per_op); 389312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 389412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_set_mult(iter, index, mult); 389512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 389612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 389712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 389812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 389912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 390012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_free_chan_resources - free the resources allocated 390112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 390212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_free_chan_resources(struct dma_chan *chan) 390312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 390412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 390512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter, *_iter; 390612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int in_use_descs = 0; 390712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 390812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 390912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_slot_cleanup(ppc440spe_chan); 391012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 391112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 391212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain, 391312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 391412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin in_use_descs++; 391512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&iter->chain_node); 391612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 391712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe_reverse(iter, _iter, 391812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &ppc440spe_chan->all_slots, slot_node) { 391912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&iter->slot_node); 392012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(iter); 392112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->slots_allocated--; 392212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 392312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->last_used = NULL; 392412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 392512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 392612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d %s slots_allocated %d\n", 392712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, 392812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, ppc440spe_chan->slots_allocated); 392912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 393012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 393112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* one is ok since we left it on there on purpose */ 393212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (in_use_descs > 1) 393312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n", 393412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin in_use_descs - 1); 393512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 393612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 393712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 39380793448187643b50af89d36b08470baf45a3cab4Linus Walleij * ppc440spe_adma_tx_status - poll the status of an ADMA transaction 393912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @chan: ADMA channel handle 394012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @cookie: ADMA transaction identifier 39410793448187643b50af89d36b08470baf45a3cab4Linus Walleij * @txstate: a holder for the current state of the channel 394212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 39430793448187643b50af89d36b08470baf45a3cab4Linus Walleijstatic enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan, 39440793448187643b50af89d36b08470baf45a3cab4Linus Walleij dma_cookie_t cookie, struct dma_tx_state *txstate) 394512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 394612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 394712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t last_used; 394812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t last_complete; 394912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin enum dma_status ret; 395012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 395112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 395212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin last_used = chan->cookie; 395312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin last_complete = ppc440spe_chan->completed_cookie; 395412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 3955bca3469205402d9fb14060d255d8786ae2256640Dan Williams dma_set_tx_state(txstate, last_complete, last_used, 0); 395612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 395712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = dma_async_is_complete(cookie, last_complete, last_used); 395812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret == DMA_SUCCESS) 395912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 396012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 396112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_slot_cleanup(ppc440spe_chan); 396212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 396312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin last_used = chan->cookie; 396412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin last_complete = ppc440spe_chan->completed_cookie; 396512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 3966bca3469205402d9fb14060d255d8786ae2256640Dan Williams dma_set_tx_state(txstate, last_complete, last_used, 0); 396712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 396812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return dma_async_is_complete(cookie, last_complete, last_used); 396912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 397012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 397112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 397212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_eot_handler - end of transfer interrupt handler 397312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 397412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data) 397512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 397612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan = data; 397712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 397812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, 397912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s\n", chan->device->id, __func__); 398012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 398112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_schedule(&chan->irq_tasklet); 398212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_device_clear_eot_status(chan); 398312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 398412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return IRQ_HANDLED; 398512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 398612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 398712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 398812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_err_handler - DMA error interrupt handler; 398912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * do the same things as a eot handler 399012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 399112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic irqreturn_t ppc440spe_adma_err_handler(int irq, void *data) 399212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 399312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan = data; 399412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 399512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, 399612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s\n", chan->device->id, __func__); 399712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 399812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_schedule(&chan->irq_tasklet); 399912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_device_clear_eot_status(chan); 400012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 400112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return IRQ_HANDLED; 400212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 400312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 400412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 400512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_test_callback - called when test operation has been done 400612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 400712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_test_callback(void *unused) 400812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 400912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin complete(&ppc440spe_r6_test_comp); 401012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 401112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 401212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 401312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w 401412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 401512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_issue_pending(struct dma_chan *chan) 401612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 401712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 401812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 401912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 402012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 402112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id, 402212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, ppc440spe_chan->pending); 402312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 402412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_chan->pending) { 402512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->pending = 0; 402612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_append(ppc440spe_chan); 402712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 402812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 402912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 403012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 403112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines 403212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * use FIFOs (as opposite to chains used in XOR) so this is a XOR 403312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * specific operation) 403412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 403512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan) 403612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 403712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 403812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t cookie; 403912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op; 404012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 404112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, 404212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s\n", chan->device->id, __func__); 404312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 404412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&chan->lock); 404512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op); 404612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op); 404712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 404812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 404912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_splice_init(&sw_desc->group_list, &chan->chain); 405012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin async_tx_ack(&sw_desc->async_tx); 405112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_null_xor(group_start); 405212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 405312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = chan->common.cookie; 405412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie++; 405512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cookie <= 1) 405612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = 2; 405712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 405812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* initialize the completed cookie to be less than 405912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the most recently used cookie 406012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 406112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->completed_cookie = cookie - 1; 406212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->common.cookie = sw_desc->async_tx.cookie = cookie; 406312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 406412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* channel should not be busy */ 406512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(ppc440spe_chan_is_busy(chan)); 406612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 406712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set the descriptor address */ 406812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc); 406912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 407012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* run the descriptor */ 407112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_run(chan); 407212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 407312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "ppc440spe adma%d" 407412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " failed to allocate null descriptor\n", 407512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id); 407612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 407712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 407812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 407912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 408012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully. 408112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * For this we just perform one WXOR operation with the same source 408212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * and destination addresses, the GF-multiplier is 1; so if RAID-6 408312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * capabilities are enabled then we'll get src/dst filled with zero. 408412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 408512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan) 408612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 408712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *iter; 408812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct page *pg; 408912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin char *a; 409012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t dma_addr, addrs[2]; 409112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 409212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int rval = 0; 409312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 409412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_WXOR, &op); 409512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 409612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pg = alloc_page(GFP_KERNEL); 409712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!pg) 409812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENOMEM; 409912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 410012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&chan->lock); 410112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1); 410212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 410312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 1 src, 1 dsr, int_ena, WXOR */ 410412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op); 410512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 410612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE); 410712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = PAGE_SIZE; 410812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 410912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 411012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rval = -EFAULT; 411112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 411212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto exit; 411312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 411412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 411512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 411612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Fill the test page with ones */ 411712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(page_address(pg), 0xFF, PAGE_SIZE); 411812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr = dma_map_page(chan->device->dev, pg, 0, 411912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 412012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 412112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup addresses */ 412212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0); 412312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0); 412412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addrs[0] = dma_addr; 412512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addrs[1] = 0; 412612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q); 412712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 412812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin async_tx_ack(&sw_desc->async_tx); 412912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.callback = ppc440spe_test_callback; 413012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.callback_param = NULL; 413112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 413212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin init_completion(&ppc440spe_r6_test_comp); 413312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 413412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_tx_submit(&sw_desc->async_tx); 413512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_issue_pending(&chan->common); 413612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 413712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin wait_for_completion(&ppc440spe_r6_test_comp); 413812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 413912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Now check if the test page is zeroed */ 414012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin a = page_address(pg); 414112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) { 414212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* page is zero - RAID-6 enabled */ 414312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rval = 0; 414412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 414512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RAID-6 was not enabled */ 414612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rval = -EINVAL; 414712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 414812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinexit: 414912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(pg); 415012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return rval; 415112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 415212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 415312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev) 415412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 415512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (adev->id) { 415612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 415712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 415812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_MEMCPY, adev->common.cap_mask); 415912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask); 416012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_MEMSET, adev->common.cap_mask); 416112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_PQ, adev->common.cap_mask); 416212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask); 416312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask); 416412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 416512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 416612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_XOR, adev->common.cap_mask); 416712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_PQ, adev->common.cap_mask); 416812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask); 416912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.cap_mask = adev->common.cap_mask; 417012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 417112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 417212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 417312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set base routines */ 417412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_alloc_chan_resources = 417512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_alloc_chan_resources; 417612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_free_chan_resources = 417712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_free_chan_resources; 41780793448187643b50af89d36b08470baf45a3cab4Linus Walleij adev->common.device_tx_status = ppc440spe_adma_tx_status; 417912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_issue_pending = ppc440spe_adma_issue_pending; 418012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 418112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set prep routines based on capability */ 418212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) { 418312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_memcpy = 418412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_memcpy; 418512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 418612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_MEMSET, adev->common.cap_mask)) { 418712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_memset = 418812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_memset; 418912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 419012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) { 419112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_xor = XOR_MAX_OPS; 419212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_xor = 419312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_xor; 419412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 419512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) { 419612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (adev->id) { 419712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 419812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_set_maxpq(&adev->common, 419912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0); 420012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 420112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 420212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_set_maxpq(&adev->common, 420312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0); 420412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 420512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 420612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_pq = XOR_MAX_OPS * 3; 420712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 420812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 420912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_pq = 421012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_pq; 421112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 421212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) { 421312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (adev->id) { 421412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 421512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_pq = DMA0_FIFO_SIZE / 421612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sizeof(struct dma_cdb); 421712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 421812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 421912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_pq = DMA1_FIFO_SIZE / 422012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sizeof(struct dma_cdb); 422112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 422212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 422312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_pq_val = 422412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_pqzero_sum; 422512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 422612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) { 422712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (adev->id) { 422812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 422912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_xor = DMA0_FIFO_SIZE / 423012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sizeof(struct dma_cdb); 423112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 423212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 423312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_xor = DMA1_FIFO_SIZE / 423412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sizeof(struct dma_cdb); 423512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 423612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 423712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_xor_val = 423812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_xor_zero_sum; 423912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 424012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) { 424112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_interrupt = 424212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_interrupt; 424312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 424412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: " 424512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "( %s%s%s%s%s%s%s)\n", 424612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_name(adev->dev), 424712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "", 424812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "", 424912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "", 425012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "", 425112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "", 425212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_MEMSET, adev->common.cap_mask) ? "memset " : "", 425312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : ""); 425412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 425512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 425612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev, 425712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 425812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int *initcode) 425912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 42602dc11581376829303b98eadb2de253bee065a56aGrant Likely struct platform_device *ofdev; 426112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct device_node *np; 426212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int ret; 426312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 42642dc11581376829303b98eadb2de253bee065a56aGrant Likely ofdev = container_of(adev->dev, struct platform_device, dev); 42653e6b02d9f5a9715f7d4ff9e0978e5f9cef53d31fDan Williams np = ofdev->dev.of_node; 426612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id != PPC440SPE_XOR_ID) { 426712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->err_irq = irq_of_parse_and_map(np, 1); 426812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->err_irq == NO_IRQ) { 426912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_warn(adev->dev, "no err irq resource?\n"); 427012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *initcode = PPC_ADMA_INIT_IRQ2; 427112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->err_irq = -ENXIO; 427212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 427312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin atomic_inc(&ppc440spe_adma_err_irq_ref); 427412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 427512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->err_irq = -ENXIO; 427612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 427712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 427812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->irq = irq_of_parse_and_map(np, 0); 427912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->irq == NO_IRQ) { 428012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(adev->dev, "no irq resource\n"); 428112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *initcode = PPC_ADMA_INIT_IRQ1; 428212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENXIO; 428312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_irq_map; 428412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 428512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(adev->dev, "irq %d, err irq %d\n", 428612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->irq, adev->err_irq); 428712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 428812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = request_irq(adev->irq, ppc440spe_adma_eot_handler, 428912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 0, dev_driver_string(adev->dev), chan); 429012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) { 429112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(adev->dev, "can't request irq %d\n", 429212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->irq); 429312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *initcode = PPC_ADMA_INIT_IRQ1; 429412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -EIO; 429512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_req1; 429612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 429712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 429812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* only DMA engines have a separate error IRQ 429912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * so it's Ok if err_irq < 0 in XOR engine case. 430012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 430112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->err_irq > 0) { 430212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* both DMA engines share common error IRQ */ 430312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = request_irq(adev->err_irq, 430412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_err_handler, 430512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin IRQF_SHARED, 430612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_driver_string(adev->dev), 430712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan); 430812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) { 430912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(adev->dev, "can't request irq %d\n", 431012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->err_irq); 431112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *initcode = PPC_ADMA_INIT_IRQ2; 431212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -EIO; 431312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_req2; 431412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 431512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 431612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 431712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) { 431812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* enable XOR engine interrupts */ 431912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT | 432012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT, 432112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &adev->xor_reg->ier); 432212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 432312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 mask, enable; 432412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 432512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe"); 432612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!np) { 432712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't find I2O device tree node\n", 432812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 432912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 433012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_req2; 433112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 433212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->i2o_reg = of_iomap(np, 0); 433312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!adev->i2o_reg) { 433412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to map I2O registers\n", __func__); 433512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 433612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -EINVAL; 433712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_req2; 433812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 433912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 434012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Unmask 'CS FIFO Attention' interrupts and 434112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * enable generating interrupts on errors 434212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 434312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin enable = (adev->id == PPC440SPE_DMA0_ID) ? 434412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) : 434512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM); 434612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mask = ioread32(&adev->i2o_reg->iopim) & enable; 434712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(mask, &adev->i2o_reg->iopim); 434812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 434912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 435012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 435112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_req2: 435212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin free_irq(adev->irq, chan); 435312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_req1: 435412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin irq_dispose_mapping(adev->irq); 435512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_irq_map: 435612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->err_irq > 0) { 435712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) 435812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin irq_dispose_mapping(adev->err_irq); 435912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 436012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 436112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 436212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 436312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev, 436412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 436512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 436612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 mask, disable; 436712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 436812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) { 436912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* disable XOR engine interrupts */ 437012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mask = ioread32be(&adev->xor_reg->ier); 437112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT | 437212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT); 437312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(mask, &adev->xor_reg->ier); 437412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 437512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* disable DMAx engine interrupts */ 437612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin disable = (adev->id == PPC440SPE_DMA0_ID) ? 437712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) : 437812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM); 437912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mask = ioread32(&adev->i2o_reg->iopim) | disable; 438012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(mask, &adev->i2o_reg->iopim); 438112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 438212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin free_irq(adev->irq, chan); 438312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin irq_dispose_mapping(adev->irq); 438412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->err_irq > 0) { 438512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin free_irq(adev->err_irq, chan); 438612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) { 438712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin irq_dispose_mapping(adev->err_irq); 438812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->i2o_reg); 438912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 439012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 439112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 439212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 439312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 439412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_probe - probe the asynch device 439512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 43962dc11581376829303b98eadb2de253bee065a56aGrant Likelystatic int __devinit ppc440spe_adma_probe(struct platform_device *ofdev, 439712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const struct of_device_id *match) 439812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 439905c02542c20aa00dc9a66f4bfb1a89d1131457f2Anatolij Gustschin struct device_node *np = ofdev->dev.of_node; 440012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct resource res; 440112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_device *adev; 440212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 440312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc_dma_chan_ref *ref, *_ref; 440412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int ret = 0, initcode = PPC_ADMA_INIT_OK; 440512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const u32 *idx; 440612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int len; 440712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin void *regs; 440812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 id, pool_size; 440912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 441012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (of_device_is_compatible(np, "amcc,xor-accelerator")) { 441112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin id = PPC440SPE_XOR_ID; 441212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* As far as the XOR engine is concerned, it does not 441312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * use FIFOs but uses linked list. So there is no dependency 441412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * between pool size to allocate and the engine configuration. 441512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 441612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pool_size = PAGE_SIZE << 1; 441712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 441812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* it is DMA0 or DMA1 */ 441912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idx = of_get_property(np, "cell-index", &len); 442012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!idx || (len != sizeof(u32))) { 442112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "Device node %s has missing " 442212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "or invalid cell-index property\n", 442312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np->full_name); 442412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 442512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 442612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin id = *idx; 442712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA0,1 engines use FIFO to maintain CDBs, so we 442812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * should allocate the pool accordingly to size of this 442912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * FIFO. Thus, the pool size depends on the FIFO depth: 443012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * how much CDBs pointers the FIFO may contain then so 443112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * much CDBs we should provide in the pool. 443212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * That is 443312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * CDB size = 32B; 443412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * CDBs number = (DMA0_FIFO_SIZE >> 3); 443512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Pool size = CDBs number * CDB size = 443612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2. 443712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 443812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pool_size = (id == PPC440SPE_DMA0_ID) ? 443912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA0_FIFO_SIZE : DMA1_FIFO_SIZE; 444012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pool_size <<= 2; 444112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 444212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 444312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (of_address_to_resource(np, 0, &res)) { 444412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to get memory resource\n"); 444512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_MEMRES; 444612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 444712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out; 444812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 444912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 445012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!request_mem_region(res.start, resource_size(&res), 445112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_driver_string(&ofdev->dev))) { 4452a584bff5efae8c1d026e3a930e3d13a90264fafcJoe Perches dev_err(&ofdev->dev, "failed to request memory region %pR\n", 4453a584bff5efae8c1d026e3a930e3d13a90264fafcJoe Perches &res); 445412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_MEMREG; 445512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -EBUSY; 445612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out; 445712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 445812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 445912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* create a device */ 446012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev = kzalloc(sizeof(*adev), GFP_KERNEL); 446112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!adev) { 446212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to allocate device\n"); 446312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_ALLOC; 446412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 446512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_adev_alloc; 446612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 446712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 446812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->id = id; 446912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->pool_size = pool_size; 447012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* allocate coherent memory for hardware descriptors */ 447112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev, 447212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->pool_size, &adev->dma_desc_pool, 447312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin GFP_KERNEL); 447412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->dma_desc_pool_virt == NULL) { 447512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent " 447612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "memory for hardware descriptors\n", 447712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->pool_size); 447812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_COHERENT; 447912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 448012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_dma_alloc; 448112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 448212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(&ofdev->dev, "allocted descriptor pool virt 0x%p phys 0x%llx\n", 448312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool); 448412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 448512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin regs = ioremap(res.start, resource_size(&res)); 448612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!regs) { 448712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to ioremap regs!\n"); 448812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_regs_alloc; 448912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 449012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 449112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) { 449212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->xor_reg = regs; 449312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Reset XOR */ 449412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr); 449512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr); 449612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 449712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ? 449812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA0_FIFO_SIZE : DMA1_FIFO_SIZE; 449912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_reg = regs; 450012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMAx_FIFO_SIZE is defined in bytes, 450112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * <fsiz> - is defined in number of CDB pointers (8byte). 450212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * DMA FIFO Length = CSlength + CPlength, where 450312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * CSlength = CPlength = (fsiz + 1) * 8. 450412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 450512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2), 450612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &adev->dma_reg->fsiz); 450712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Configure DMA engine */ 450812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN, 450912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &adev->dma_reg->cfg); 451012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Clear Status */ 451112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(~0, &adev->dma_reg->dsts); 451212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 451312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 451412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dev = &ofdev->dev; 451512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.dev = &ofdev->dev; 451612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&adev->common.channels); 451712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_set_drvdata(&ofdev->dev, adev); 451812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 451912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* create a channel */ 452012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = kzalloc(sizeof(*chan), GFP_KERNEL); 452112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!chan) { 452212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "can't allocate channel structure\n"); 452312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_CHANNEL; 452412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 452512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_chan_alloc; 452612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 452712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 452812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_init(&chan->lock); 452912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&chan->chain); 453012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&chan->all_slots); 453112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device = adev; 453212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->common.device = &adev->common; 453312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_add_tail(&chan->common.device_node, &adev->common.channels); 453412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_init(&chan->irq_tasklet, ppc440spe_adma_tasklet, 453512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (unsigned long)chan); 453612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 453712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* allocate and map helper pages for async validation or 453812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * async_mult/async_sum_product operations on DMA0/1. 453912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 454012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id != PPC440SPE_XOR_ID) { 454112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->pdest_page = alloc_page(GFP_KERNEL); 454212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->qdest_page = alloc_page(GFP_KERNEL); 454312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!chan->pdest_page || 454412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin !chan->qdest_page) { 454512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan->pdest_page) 454612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(chan->pdest_page); 454712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan->qdest_page) 454812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(chan->qdest_page); 454912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 455012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_page_alloc; 455112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 455212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0, 455312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 455412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0, 455512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 455612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 455712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 455812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ref = kmalloc(sizeof(*ref), GFP_KERNEL); 455912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ref) { 456012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ref->chan = &chan->common; 456112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&ref->node); 456212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_add_tail(&ref->node, &ppc440spe_adma_chan_list); 456312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 456412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to allocate channel reference!\n"); 456512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 456612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_ref_alloc; 456712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 456812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 456912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode); 457012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) 457112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_irq; 457212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 457312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_init_capabilities(adev); 457412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 457512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = dma_async_device_register(&adev->common); 457612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) { 457712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_REGISTER; 457812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to register dma device\n"); 457912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_dev_reg; 458012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 458112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 458212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out; 458312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 458412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_dev_reg: 458512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_release_irqs(adev, chan); 458612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_irq: 458712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) { 458812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan == to_ppc440spe_adma_chan(ref->chan)) { 458912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&ref->node); 459012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ref); 459112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 459212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 459312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_ref_alloc: 459412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id != PPC440SPE_XOR_ID) { 459512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(&ofdev->dev, chan->pdest, 459612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 459712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(&ofdev->dev, chan->qdest, 459812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 459912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(chan->pdest_page); 460012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(chan->qdest_page); 460112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 460212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_page_alloc: 460312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(chan); 460412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_chan_alloc: 460512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) 460612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->xor_reg); 460712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 460812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->dma_reg); 460912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_regs_alloc: 461012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_free_coherent(adev->dev, adev->pool_size, 461112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool_virt, 461212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool); 461312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_dma_alloc: 461412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(adev); 461512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_adev_alloc: 461612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin release_mem_region(res.start, resource_size(&res)); 461712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout: 461812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (id < PPC440SPE_ADMA_ENGINES_NUM) 461912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_devices[id] = initcode; 462012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 462112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 462212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 462312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 462412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 462512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_remove - remove the asynch device 462612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 46272dc11581376829303b98eadb2de253bee065a56aGrant Likelystatic int __devexit ppc440spe_adma_remove(struct platform_device *ofdev) 462812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 462912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_device *adev = dev_get_drvdata(&ofdev->dev); 463005c02542c20aa00dc9a66f4bfb1a89d1131457f2Anatolij Gustschin struct device_node *np = ofdev->dev.of_node; 463112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct resource res; 463212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, *_chan; 463312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc_dma_chan_ref *ref, *_ref; 463412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 463512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 463612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_set_drvdata(&ofdev->dev, NULL); 463712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id < PPC440SPE_ADMA_ENGINES_NUM) 463812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_devices[adev->id] = -1; 463912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 464012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_async_device_unregister(&adev->common); 464112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 464212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(chan, _chan, &adev->common.channels, 464312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin device_node) { 464412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 464512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_release_irqs(adev, ppc440spe_chan); 464612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_kill(&ppc440spe_chan->irq_tasklet); 464712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id != PPC440SPE_XOR_ID) { 464812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest, 464912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 465012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest, 465112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 465212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(ppc440spe_chan->pdest_page); 465312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(ppc440spe_chan->qdest_page); 465412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 465512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, 465612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin node) { 465712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_chan == 465812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin to_ppc440spe_adma_chan(ref->chan)) { 465912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&ref->node); 466012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ref); 466112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 466212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 466312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&chan->device_node); 466412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ppc440spe_chan); 466512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 466612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 466712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_free_coherent(adev->dev, adev->pool_size, 466812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool_virt, adev->dma_desc_pool); 466912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) 467012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->xor_reg); 467112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 467212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->dma_reg); 467312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_address_to_resource(np, 0, &res); 467412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin release_mem_region(res.start, resource_size(&res)); 467512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(adev); 467612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 467712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 467812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 467912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* 468012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * /sys driver interface to enable h/w RAID-6 capabilities 468112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/ 468212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * directory are "devices", "enable" and "poly". 468312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * "devices" shows available engines. 468412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * "enable" is used to enable RAID-6 capabilities or to check 468512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * whether these has been activated. 468612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * "poly" allows setting/checking used polynomial (for PPC440SPe only). 468712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 468812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 468912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t show_ppc440spe_devices(struct device_driver *dev, char *buf) 469012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 469112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ssize_t size = 0; 469212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 469312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 469412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) { 469512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_adma_devices[i] == -1) 469612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 469712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size += snprintf(buf + size, PAGE_SIZE - size, 469812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "PPC440SP(E)-ADMA.%d: %s\n", i, 469912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc_adma_errors[ppc440spe_adma_devices[i]]); 470012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 470112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return size; 470212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 470312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 470412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t show_ppc440spe_r6enable(struct device_driver *dev, char *buf) 470512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 470612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return snprintf(buf, PAGE_SIZE, 470712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "PPC440SP(e) RAID-6 capabilities are %sABLED.\n", 470812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_r6_enabled ? "EN" : "DIS"); 470912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 471012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 471112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t store_ppc440spe_r6enable(struct device_driver *dev, 471212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const char *buf, size_t count) 471312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 471412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long val; 471512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 471612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!count || count > 11) 471712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 471812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 471912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!ppc440spe_r6_tchan) 472012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EFAULT; 472112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 472212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Write a key */ 472312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sscanf(buf, "%lx", &val); 472412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val); 472512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin isync(); 472612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 472712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Verify whether it really works now */ 472812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) { 472912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_info("PPC440SP(e) RAID-6 has been activated " 473012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "successfully\n"); 473112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_r6_enabled = 1; 473212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 473312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_info("PPC440SP(e) RAID-6 hasn't been activated!" 473412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " Error key ?\n"); 473512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_r6_enabled = 0; 473612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 473712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return count; 473812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 473912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 474012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t show_ppc440spe_r6poly(struct device_driver *dev, char *buf) 474112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 474212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ssize_t size = 0; 474312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 reg; 474412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 474512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#ifdef CONFIG_440SP 474612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 440SP has fixed polynomial */ 474712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg = 0x4d; 474812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#else 474912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL); 475012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg >>= MQ0_CFBHL_POLY; 475112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg &= 0xFF; 475212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#endif 475312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 475412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver " 475512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "uses 0x1%02x polynomial.\n", reg); 475612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return size; 475712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 475812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 475912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t store_ppc440spe_r6poly(struct device_driver *dev, 476012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const char *buf, size_t count) 476112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 476212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long reg, val; 476312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 476412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#ifdef CONFIG_440SP 476512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 440SP uses default 0x14D polynomial only */ 476612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 476712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#endif 476812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 476912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!count || count > 6) 477012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 477112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 477212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* e.g., 0x14D or 0x11D */ 477312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sscanf(buf, "%lx", &val); 477412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 477512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (val & ~0x1FF) 477612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 477712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 477812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin val &= 0xFF; 477912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL); 478012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg &= ~(0xFF << MQ0_CFBHL_POLY); 478112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg |= val << MQ0_CFBHL_POLY; 478212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg); 478312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 478412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return count; 478512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 478612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 478712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic DRIVER_ATTR(devices, S_IRUGO, show_ppc440spe_devices, NULL); 478812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic DRIVER_ATTR(enable, S_IRUGO | S_IWUSR, show_ppc440spe_r6enable, 478912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin store_ppc440spe_r6enable); 479012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic DRIVER_ATTR(poly, S_IRUGO | S_IWUSR, show_ppc440spe_r6poly, 479112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin store_ppc440spe_r6poly); 479212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 479312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* 479412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Common initialisation for RAID engines; allocate memory for 479512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * DMAx FIFOs, perform configuration common for all DMA engines. 479612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Further DMA engine specific configuration is done at probe time. 479712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 479812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_configure_raid_devices(void) 479912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 480012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct device_node *np; 480112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct resource i2o_res; 480212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct i2o_regs __iomem *i2o_reg; 480312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_host_t i2o_dcr_host; 480412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int dcr_base, dcr_len; 480512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i, ret; 480612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 480712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe"); 480812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!np) { 480912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't find I2O device tree node\n", 481012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 481112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENODEV; 481212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 481312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 481412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (of_address_to_resource(np, 0, &i2o_res)) { 481512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 481612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 481712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 481812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 481912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i2o_reg = of_iomap(np, 0); 482012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!i2o_reg) { 482112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to map I2O registers\n", __func__); 482212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 482312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 482412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 482512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 482612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Get I2O DCRs base */ 482712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_base = dcr_resource_start(np, 0); 482812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_len = dcr_resource_len(np, 0); 482912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!dcr_base && !dcr_len) { 483012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't get DCR registers base/len!\n", 483112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np->full_name); 483212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 483312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(i2o_reg); 483412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENODEV; 483512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 483612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 483712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i2o_dcr_host = dcr_map(np, dcr_base, dcr_len); 483812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!DCR_MAP_OK(i2o_dcr_host)) { 483912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to map DCRs!\n", np->full_name); 484012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 484112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(i2o_reg); 484212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENODEV; 484312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 484412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 484512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 484612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share 484712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the base address of FIFO memory space. 484812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Actually we need twice more physical memory than programmed in the 484912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * <fsiz> register (because there are two FIFOs for each DMA: CP and CS) 485012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 485112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1, 485212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin GFP_KERNEL); 485312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!ppc440spe_dma_fifo_buf) { 485412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__); 485512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(i2o_reg); 485612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_unmap(i2o_dcr_host, dcr_len); 485712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENOMEM; 485812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 485912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 486012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 486112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Configure h/w 486212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 486312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Reset I2O/DMA */ 486412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA); 486512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mtdcri(SDR0, DCRN_SDR0_SRST, 0); 486612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 486712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup the base address of mmaped registers */ 486812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32)); 486912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) | 487012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin I2O_REG_ENABLE); 487112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_unmap(i2o_dcr_host, dcr_len); 487212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 487312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup FIFO memory space base address */ 487412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(0, &i2o_reg->ifbah); 487512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal); 487612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 487712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set zero FIFO size for I2O, so the whole 487812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma_fifo_buf is used by DMAs. 487912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * DMAx_FIFOs will be configured while probe. 488012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 488112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(0, &i2o_reg->ifsiz); 488212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(i2o_reg); 488312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 488412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* To prepare WXOR/RXOR functionality we need access to 488512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Memory Queue Module DCRs (finally it will be enabled 488612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * via /sys interface of the ppc440spe ADMA driver). 488712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 488812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe"); 488912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!np) { 489012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't find MQ device tree node\n", 489112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 489212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 489312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_free; 489412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 489512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 489612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Get MQ DCRs base */ 489712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_base = dcr_resource_start(np, 0); 489812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_len = dcr_resource_len(np, 0); 489912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!dcr_base && !dcr_len) { 490012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't get DCR registers base/len!\n", 490112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np->full_name); 490212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 490312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_mq; 490412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 490512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 490612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len); 490712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) { 490812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to map DCRs!\n", np->full_name); 490912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 491012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_mq; 491112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 491212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 491312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_mq_dcr_len = dcr_len; 491412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 491512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set HB alias */ 491612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB); 491712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 491812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set: 491912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - LL transaction passing limit to 1; 492012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - Memory controller cycle limit to 1; 492112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - Galois Polynomial to 0x14d (default) 492212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 492312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, 492412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) | 492512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY)); 492612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 492712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin atomic_set(&ppc440spe_adma_err_irq_ref, 0); 492812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) 492912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_devices[i] = -1; 493012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 493112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 493212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 493312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_mq: 493412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 493512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_free: 493612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ppc440spe_dma_fifo_buf); 493712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 493812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 493912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 49404b1cf1facca31b7db2a61d8aa2ba40d5a93a0957Márton Némethstatic const struct of_device_id ppc440spe_adma_of_match[] __devinitconst = { 494112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin { .compatible = "ibm,dma-440spe", }, 494212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin { .compatible = "amcc,xor-accelerator", }, 494312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin {}, 494412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 494512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinMODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match); 494612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 494712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct of_platform_driver ppc440spe_adma_driver = { 494812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin .probe = ppc440spe_adma_probe, 494912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin .remove = __devexit_p(ppc440spe_adma_remove), 495012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin .driver = { 495112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin .name = "PPC440SP(E)-ADMA", 495212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin .owner = THIS_MODULE, 49534018294b53d1dae026880e45f174c1cc63b5d435Grant Likely .of_match_table = ppc440spe_adma_of_match, 495412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin }, 495512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 495612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 495712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic __init int ppc440spe_adma_init(void) 495812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 495912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int ret; 496012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 496112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = ppc440spe_configure_raid_devices(); 496212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) 496312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 496412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 496512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = of_register_platform_driver(&ppc440spe_adma_driver); 496612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) { 496712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to register platform driver\n", 496812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 496912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_reg; 497012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 497112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 497212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Initialization status */ 497312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = driver_create_file(&ppc440spe_adma_driver.driver, 497412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_devices); 497512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) 497612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_dev; 497712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 497812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RAID-6 h/w enable entry */ 497912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = driver_create_file(&ppc440spe_adma_driver.driver, 498012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_enable); 498112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) 498212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_en; 498312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 498412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* GF polynomial to use */ 498512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = driver_create_file(&ppc440spe_adma_driver.driver, 498612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_poly); 498712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!ret) 498812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 498912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 499012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 499112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_enable); 499212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_en: 499312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 499412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_devices); 499512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_dev: 499612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* User will not be able to enable h/w RAID-6 */ 499712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to create RAID-6 driver interface\n", 499812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 499912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_unregister_platform_driver(&ppc440spe_adma_driver); 500012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_reg: 500112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len); 500212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ppc440spe_dma_fifo_buf); 500312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 500412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 500512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 500612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void __exit ppc440spe_adma_exit(void) 500712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 500812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 500912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_poly); 501012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 501112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_enable); 501212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 501312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_devices); 501412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_unregister_platform_driver(&ppc440spe_adma_driver); 501512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len); 501612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ppc440spe_dma_fifo_buf); 501712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 501812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 501912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinarch_initcall(ppc440spe_adma_init); 502012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinmodule_exit(ppc440spe_adma_exit); 502112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 502212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinMODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>"); 502312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinMODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver"); 502412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinMODULE_LICENSE("GPL"); 5025