adma.c revision d38a8c622a1b382336c3e152c6caf4e11d1f1b2a
112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* 212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Copyright (C) 2006-2009 DENX Software Engineering. 312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Author: Yuri Tikhonov <yur@emcraft.com> 512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Further porting to arch/powerpc by 712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Anatolij Gustschin <agust@denx.de> 812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * This program is free software; you can redistribute it and/or modify it 1012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * under the terms of the GNU General Public License as published by the Free 1112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Software Foundation; either version 2 of the License, or (at your option) 1212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * any later version. 1312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 1412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * This program is distributed in the hope that it will be useful, but WITHOUT 1512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 1612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 1712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * more details. 1812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 1912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * You should have received a copy of the GNU General Public License along with 2012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * this program; if not, write to the Free Software Foundation, Inc., 59 2112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Temple Place - Suite 330, Boston, MA 02111-1307, USA. 2212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 2312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * The full GNU General Public License is included in this distribution in the 2412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * file called COPYING. 2512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 2612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 2712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* 2812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * This driver supports the asynchrounous DMA copy and RAID engines available 2912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * on the AMCC PPC440SPe Processors. 3012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) 3112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ADMA driver written by D.Williams. 3212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 3312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 3412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/init.h> 3512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/module.h> 3612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/async_tx.h> 3712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/delay.h> 3812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/dma-mapping.h> 3912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/spinlock.h> 4012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/interrupt.h> 415a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 4212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/uaccess.h> 4312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/proc_fs.h> 4412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/of.h> 4512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/of_platform.h> 4612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <asm/dcr.h> 4712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <asm/dcr-regs.h> 4812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include "adma.h" 49d2ebfb335b0426deb1a4fb14e4e926d81ecd8235Russell King - ARM Linux#include "../dmaengine.h" 5012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 5112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinenum ppc_adma_init_code { 5212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_OK = 0, 5312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_MEMRES, 5412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_MEMREG, 5512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_ALLOC, 5612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_COHERENT, 5712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_CHANNEL, 5812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_IRQ1, 5912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_IRQ2, 6012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_REGISTER 6112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 6212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 6312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic char *ppc_adma_errors[] = { 6412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_OK] = "ok", 6512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource", 6612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_MEMREG] = "failed to request memory region", 6712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev " 6812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "structure", 6912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for " 7012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "hardware descriptors", 7112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel", 7212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_IRQ1] = "failed to request first irq", 7312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_IRQ2] = "failed to request second irq", 7412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device", 7512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 7612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 7712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic enum ppc_adma_init_code 7812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM]; 7912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 8012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstruct ppc_dma_chan_ref { 8112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan; 8212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct list_head node; 8312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 8412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 8512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* The list of channels exported by ppc440spe ADMA */ 8612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstruct list_head 8712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list); 8812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 8912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* This flag is set when want to refetch the xor chain in the interrupt 9012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * handler 9112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 9212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 do_xor_refetch; 9312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 9412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* Pointer to DMA0, DMA1 CP/CS FIFO */ 9512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void *ppc440spe_dma_fifo_buf; 9612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 9712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* Pointers to last submitted to DMA0, DMA1 CDBs */ 9812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *chan_last_sub[3]; 9912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *chan_first_cdb[3]; 10012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 10112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* Pointer to last linked and submitted xor CB */ 10212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *xor_last_linked; 10312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *xor_last_submit; 10412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 10512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* This array is used in data-check operations for storing a pattern */ 10612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic char ppc440spe_qword[16]; 10712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 10812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic atomic_t ppc440spe_adma_err_irq_ref; 10912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic dcr_host_t ppc440spe_mq_dcr_host; 11012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic unsigned int ppc440spe_mq_dcr_len; 11112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 11212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* Since RXOR operations use the common register (MQ0_CF2H) for setting-up 11312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the block size in transactions, then we do not allow to activate more than 11412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * only one RXOR transactions simultaneously. So use this var to store 11512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is 11612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * set) or not (PPC440SPE_RXOR_RUN is clear). 11712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 11812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic unsigned long ppc440spe_rxor_state; 11912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 12012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* These are used in enable & check routines 12112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 12212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 ppc440spe_r6_enabled; 12312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_chan *ppc440spe_r6_tchan; 12412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct completion ppc440spe_r6_test_comp; 12512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 12612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_dma2rxor_prep_src( 12712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 12812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_rxor *cursor, int index, 12912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, u32 addr); 13012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_set_src( 13112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 13212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index, dma_addr_t addr); 13312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_set_mult( 13412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 13512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index, u8 mult); 13612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 13712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#ifdef ADMA_LL_DEBUG 13812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define ADMA_LL_DBG(x) ({ if (1) x; 0; }) 13912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#else 14012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define ADMA_LL_DBG(x) ({ if (0) x; 0; }) 14112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#endif 14212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 14312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void print_cb(struct ppc440spe_adma_chan *chan, void *block) 14412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 14512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *cdb; 14612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *cb; 14712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 14812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 14912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 15012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 0: 15112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 1: 15212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb = block; 15312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 15412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("CDB at %p [%d]:\n" 15512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n" 15612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t sg1u 0x%08x sg1l 0x%08x\n" 15712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t sg2u 0x%08x sg2l 0x%08x\n" 15812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t sg3u 0x%08x sg3l 0x%08x\n", 15912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb, chan->device->id, 16012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt), 16112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l), 16212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l), 16312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l) 16412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ); 16512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 16612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 2: 16712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb = block; 16812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 16912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("CB at %p [%d]:\n" 17012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n" 17112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t cbtah 0x%08x cbtal 0x%08x\n" 17212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t cblah 0x%08x cblal 0x%08x\n", 17312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb, chan->device->id, 17412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb->cbc, cb->cbbc, cb->cbs, 17512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb->cbtah, cb->cbtal, 17612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb->cblah, cb->cblal); 17712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < 16; i++) { 17812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i && !cb->ops[i].h && !cb->ops[i].l) 17912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 18012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n", 18112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i, cb->ops[i].h, cb->ops[i].l); 18212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 18312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 18412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 18512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 18612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 18712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void print_cb_list(struct ppc440spe_adma_chan *chan, 18812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter) 18912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 19012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (; iter; iter = iter->hw_next) 19112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin print_cb(chan, iter->hw_desc); 19212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 19312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 19412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src, 19512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt) 19612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 19712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 19812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 19912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\n%s(%d):\nsrc: ", __func__, id); 20012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 20112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx ", src[i]); 20212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("dst:\n\t0x%016llx\n", dst); 20312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 20412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 20512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src, 20612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt) 20712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 20812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 20912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 21012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\n%s(%d):\nsrc: ", __func__, id); 21112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 21212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx ", src[i]); 21312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("dst: "); 21412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < 2; i++) 21512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx ", dst[i]); 21612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 21712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 21812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src, 21912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt, 22012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf) 22112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 22212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 22312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 22412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id); 22512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (scf) { 22612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 22712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]); 22812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 22912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 23012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx(no) ", src[i]); 23112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 23212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 23312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("dst: "); 23412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < 2; i++) 23512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx ", src[src_cnt + i]); 23612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 23712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 23812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/****************************************************************************** 23912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Command (Descriptor) Blocks low-level routines 24012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ******************************************************************************/ 24112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 24212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT 24312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * pseudo operation 24412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 24512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc, 24612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 24712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 24812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *p; 24912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 25012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 25112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 25212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin p = desc->hw_desc; 25312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 25412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* NOP with Command Block Complete Enable */ 25512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin p->cbc = XOR_CBCR_CBCE_BIT; 25612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 25712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 25812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 25912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 26012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* NOP with interrupt */ 26112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &desc->flags); 26212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 26312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 26412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id, 26512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 26612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 26712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 26812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 26912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 27012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 27112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR 27212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * pseudo operation 27312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 27412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc) 27512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 27612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 27712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 27812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = 0; 27912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = 1; 28012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 28112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 28212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 28312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation 28412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 28512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc, 28612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, unsigned long flags) 28712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 28812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *hw_desc = desc->hw_desc; 28912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 29012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 29112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 29212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = src_cnt; 29312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = 1; 29412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 29512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt; 29612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 29712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Enable interrupt on completion */ 29812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc |= XOR_CBCR_CBCE_BIT; 29912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 30012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 30112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 30212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ 30312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * operation in DMA2 controller 30412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 30512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc, 30612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int dst_cnt, int src_cnt, unsigned long flags) 30712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 30812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *hw_desc = desc->hw_desc; 30912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 31012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 31112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 31212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = src_cnt; 31312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = dst_cnt; 31412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags)); 31512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->descs_per_op = 0; 31612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 31712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc = XOR_CBCR_TGT_BIT; 31812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 31912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Enable interrupt on completion */ 32012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc |= XOR_CBCR_CBCE_BIT; 32112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 32212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 32312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define DMA_CTRL_FLAGS_LAST DMA_PREP_FENCE 32412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define DMA_PREP_ZERO_P (DMA_CTRL_FLAGS_LAST << 1) 32512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define DMA_PREP_ZERO_Q (DMA_PREP_ZERO_P << 1) 32612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 32712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 32812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation 32912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * with DMA0/1 33012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 33112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc, 33212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int dst_cnt, int src_cnt, unsigned long flags, 33312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op) 33412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 33512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 33612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 33712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 dopc; 33812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 33912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Common initialization of a PQ descriptors chain */ 34012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bits(op, &desc->flags); 34112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = src_cnt; 34212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = dst_cnt; 34312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 34412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR MULTICAST if both P and Q are being computed 34512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * MV_SG1_SG2 if Q only 34612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 34712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ? 34812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2; 34912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 35012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &desc->group_list, chain_node) { 35112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 35212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 35312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 35412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(!list_is_last(&iter->chain_node, 35512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &desc->group_list))) { 35612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 35712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 35812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 35912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 36012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 36112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is the last descriptor. 36212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * this slot will be pasted from ADMA level 36312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * each time it wants to configure parameters 36412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * of the transaction (src, dst, ...) 36512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 36612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 36712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 36812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 36912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 37012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 37112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 37212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 37312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 37412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set OPS depending on WXOR/RXOR type of operation */ 37512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) { 37612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is a WXOR only chain: 37712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - first descriptors are for zeroing destinations 37812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * if PPC440SPE_ZERO_P/Q set; 37912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - descriptors remained are for GF-XOR operations. 38012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 38112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&desc->group_list, 38212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 38312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 38412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 38512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) { 38612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 38712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 38812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 38912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 39012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 39112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 39212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 39312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) { 39412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 39512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 39612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 39712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 39812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 39912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 40012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 40112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &desc->group_list, chain_node) { 40212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 40312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = dopc; 40412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 40512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 40612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is either RXOR-only or mixed RXOR/WXOR */ 40712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 40812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* The first 1 or 2 slots in chain are always RXOR, 40912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * if need to calculate P & Q, then there are two 41012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * RXOR slots; if only P or only Q, then there is one 41112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 41212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&desc->group_list, 41312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 41412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 41512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 41612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 41712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 41812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (desc->dst_cnt == DMA_DEST_MAX_NUM) { 41912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 42012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 42112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 42212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 42312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 42412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 42512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 42612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* The remaining descs (if any) are WXORs */ 42712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) { 42812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 42912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 43012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 43112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &desc->group_list, 43212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 43312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 43412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = dopc; 43512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 43612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 43712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 43812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 43912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 44012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 44112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor 44212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for PQ_ZERO_SUM operation 44312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 44412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_dma01pqzero_sum( 44512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 44612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int dst_cnt, int src_cnt) 44712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 44812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 44912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 45012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i = 0; 45112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST : 45212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_OPC_MV_SG1_SG2; 45312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 45412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Initialize starting from 2nd or 3rd descriptor dependent 45512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * on dst_cnt. First one or two slots are for cloning P 45612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * and/or Q to chan->pdest and/or chan->qdest as we have 45712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * to preserve original P/Q. 45812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 45912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&desc->group_list, 46012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 46112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 46212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 46312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 46412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dst_cnt > 1) { 46512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 46612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 46712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 46812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* initialize each source descriptor in chain */ 46912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &desc->group_list, chain_node) { 47012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 47112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 47212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->src_cnt = 0; 47312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->dst_cnt = 0; 47412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 47512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is a ZERO_SUM operation: 47612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - <src_cnt> descriptors starting from 2nd or 3rd 47712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor are for GF-XOR operations; 47812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - remaining <dst_cnt> descriptors are for checking the result 47912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 48012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i++ < src_cnt) 48112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* MV_SG1_SG2 if only Q is being verified 48212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * MULTICAST if both P and Q are being verified 48312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 48412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = dopc; 48512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 48612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA_CDB_OPC_DCHECK128 operation */ 48712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_DCHECK128; 48812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 48912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(!list_is_last(&iter->chain_node, 49012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &desc->group_list))) { 49112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 49212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 49312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 49412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 49512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 49612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is the last descriptor. 49712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * this slot will be pasted from ADMA level 49812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * each time it wants to configure parameters 49912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * of the transaction (src, dst, ...) 50012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 50112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 50212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* always enable interrupt generation since we get 50312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the status of pqzero from the handler 50412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 50512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 50612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 50712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 50812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = src_cnt; 50912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = dst_cnt; 51012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 51112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 51212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 51312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation 51412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 51512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc, 51612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long flags) 51712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 51812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc = desc->hw_desc; 51912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 52012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 52112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 52212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = 1; 52312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = 1; 52412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 52512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 52612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &desc->flags); 52712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 52812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &desc->flags); 52912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 53012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 53112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 53212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 53312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 53412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_memset - initialize the descriptor for MEMSET operation 53512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 53612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_memset(struct ppc440spe_adma_desc_slot *desc, 53712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int value, unsigned long flags) 53812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 53912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc = desc->hw_desc; 54012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 54112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 54212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 54312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = 1; 54412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = 1; 54512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 54612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 54712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &desc->flags); 54812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 54912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &desc->flags); 55012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 55112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->sg1u = hw_desc->sg1l = cpu_to_le32((u32)value); 55212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->sg3u = hw_desc->sg3l = cpu_to_le32((u32)value); 55312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_DFILL128; 55412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 55512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 55612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 55712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_src_addr - set source address into the descriptor 55812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 55912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc, 56012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 56112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_idx, dma_addr_t addrh, 56212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addrl) 56312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 56412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 56512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 56612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin phys_addr_t addr64, tmplow, tmphi; 56712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 56812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 56912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 57012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 57112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addrh) { 57212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr64 = addrl; 57312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmphi = (addr64 >> 32); 57412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmplow = (addr64 & 0xFFFFFFFF); 57512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 57612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmphi = addrh; 57712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmplow = addrl; 57812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 57912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 58012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow); 58112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi); 58212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 58312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 58412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 58512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->ops[src_idx].l = addrl; 58612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->ops[src_idx].h |= addrh; 58712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 58812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 58912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 59012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 59112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 59212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_src_mult - set source address mult into the descriptor 59312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 59412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc, 59512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, u32 mult_index, 59612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int sg_index, unsigned char mult_value) 59712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 59812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 59912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 60012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 *psgu; 60112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 60212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 60312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 60412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 60512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 60612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 60712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (sg_index) { 60812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* for RXOR operations set multiplier 60912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * into source cued address 61012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 61112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_SG_SRC: 61212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgu = &dma_hw_desc->sg1u; 61312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 61412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* for WXOR operations set multiplier 61512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * into destination cued address(es) 61612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 61712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_SG_DST1: 61812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgu = &dma_hw_desc->sg2u; 61912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 62012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_SG_DST2: 62112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgu = &dma_hw_desc->sg3u; 62212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 62312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 62412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 62512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 62612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 62712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *psgu |= cpu_to_le32(mult_value << mult_index); 62812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 62912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 63012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 63112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 63212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 63312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 63412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 63512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 63612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 63712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 63812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_dest_addr - set destination address into the descriptor 63912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 64012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc, 64112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 64212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addrh, dma_addr_t addrl, 64312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 dst_idx) 64412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 64512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 64612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 64712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin phys_addr_t addr64, tmphi, tmplow; 64812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 *psgu, *psgl; 64912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 65012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 65112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 65212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 65312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addrh) { 65412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr64 = addrl; 65512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmphi = (addr64 >> 32); 65612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmplow = (addr64 & 0xFFFFFFFF); 65712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 65812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmphi = addrh; 65912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmplow = addrl; 66012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 66112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 66212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 66312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u; 66412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l; 66512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 66612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *psgl = cpu_to_le32((u32)tmplow); 66712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *psgu |= cpu_to_le32((u32)tmphi); 66812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 66912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 67012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 67112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbtal = addrl; 67212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbtah |= addrh; 67312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 67412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 67512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 67612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 67712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 67812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_byte_count - set number of data bytes involved 67912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * into the operation 68012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 68112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc, 68212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 68312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 byte_count) 68412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 68512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 68612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 68712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 68812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 68912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 69012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 69112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 69212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->cnt = cpu_to_le32(byte_count); 69312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 69412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 69512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 69612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbbc = byte_count; 69712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 69812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 69912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 70012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 70112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 70212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_rxor_block_size - set RXOR block size 70312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 70412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count) 70512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 70612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* assume that byte_count is aligned on the 512-boundary; 70712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * thus write it directly to the register (bits 23:31 are 70812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * reserved there). 70912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 71012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count); 71112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 71212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 71312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 71412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_dcheck - set CHECK pattern 71512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 71612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc, 71712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, u8 *qword) 71812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 71912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 72012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 72112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 72212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 72312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 72412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 72512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(qword[0], &dma_hw_desc->sg3l); 72612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(qword[4], &dma_hw_desc->sg3u); 72712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(qword[8], &dma_hw_desc->sg2l); 72812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(qword[12], &dma_hw_desc->sg2u); 72912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 73012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 73112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 73212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 73312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 73412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 73512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 73612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_xor_set_link - set link address in xor CB 73712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 73812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc, 73912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *next_desc) 74012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 74112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc = prev_desc->hw_desc; 74212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 74312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!next_desc || !(next_desc->phys))) { 74412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n", 74512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, next_desc, 74612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin next_desc ? next_desc->phys : 0); 74712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 74812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 74912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 75012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbs = 0; 75112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cblal = next_desc->phys; 75212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cblah = 0; 75312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT; 75412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 75512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 75612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 75712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_link - set the address of descriptor following this 75812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor in chain 75912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 76012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan, 76112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *prev_desc, 76212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *next_desc) 76312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 76412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long flags; 76512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *tail = next_desc; 76612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 76712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!prev_desc || !next_desc || 76812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (prev_desc->hw_next && prev_desc->hw_next != next_desc))) { 76912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* If previous next is overwritten something is wrong. 77012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * though we may refetch from append to initiate list 77112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * processing; in this case - it's ok. 77212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 77312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; " 77412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "prev->hw_next=0x%p\n", __func__, prev_desc, 77512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin next_desc, prev_desc ? prev_desc->hw_next : 0); 77612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 77712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 77812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 77912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin local_irq_save(flags); 78012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 78112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do s/w chaining both for DMA and XOR descriptors */ 78212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prev_desc->hw_next = next_desc; 78312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 78412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 78512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 78612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 78712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 78812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 78912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* bind descriptor to the chain */ 79012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (tail->hw_next) 79112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tail = tail->hw_next; 79212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_linked = tail; 79312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 79412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (prev_desc == xor_last_submit) 79512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do not link to the last submitted CB */ 79612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 79712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_xor_set_link(prev_desc, next_desc); 79812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 79912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 80012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 80112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin local_irq_restore(flags); 80212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 80312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 80412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 80512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_get_src_addr - extract the source address from the descriptor 80612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 80712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 ppc440spe_desc_get_src_addr(struct ppc440spe_adma_desc_slot *desc, 80812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, int src_idx) 80912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 81012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 81112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 81212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 81312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 81412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 81512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 81612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 81712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* May have 0, 1, 2, or 3 sources */ 81812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (dma_hw_desc->opc) { 81912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_NO_OP: 82012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DFILL128: 82112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 82212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DCHECK128: 82312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(src_idx)) { 82412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: try to get %d source for" 82512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " DCHECK128\n", __func__, src_idx); 82612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 82712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 82812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu(dma_hw_desc->sg1l); 82912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MULTICAST: 83012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MV_SG1_SG2: 83112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(src_idx > 2)) { 83212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: try to get %d source from" 83312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " DMA descr\n", __func__, src_idx); 83412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 83512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 83612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_idx) { 83712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (le32_to_cpu(dma_hw_desc->sg1u) & 83812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_WIN_MSK) { 83912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 region; 84012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 84112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_idx == 1) 84212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu( 84312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1l) + 84412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->unmap_len; 84512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 84612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin region = (le32_to_cpu( 84712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1u)) >> 84812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 84912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 85012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin region &= DMA_CUED_REGION_MSK; 85112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (region) { 85212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_RXOR123: 85312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu( 85412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1l) + 85512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (desc->unmap_len << 1); 85612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_RXOR124: 85712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu( 85812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1l) + 85912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (desc->unmap_len * 3); 86012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_RXOR125: 86112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu( 86212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1l) + 86312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (desc->unmap_len << 2); 86412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 86512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR 86612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "%s: try to" 86712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " get src3 for region %02x" 86812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "PPC440SPE_DESC_RXOR12?\n", 86912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, region); 87012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 87112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 87212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 87312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR 87412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "%s: try to get %d" 87512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " source for non-cued descr\n", 87612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, src_idx); 87712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 87812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 87912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 88012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu(dma_hw_desc->sg1l); 88112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 88212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: unknown OPC 0x%02x\n", 88312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dma_hw_desc->opc); 88412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 88512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 88612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu(dma_hw_desc->sg1l); 88712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 88812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* May have up to 16 sources */ 88912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 89012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return xor_hw_desc->ops[src_idx].l; 89112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 89212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 89312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 89412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 89512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 89612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_get_dest_addr - extract the destination address from the 89712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor 89812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 89912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 ppc440spe_desc_get_dest_addr(struct ppc440spe_adma_desc_slot *desc, 90012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, int idx) 90112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 90212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 90312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 90412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 90512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 90612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 90712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 90812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 90912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 91012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(!idx)) 91112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu(dma_hw_desc->sg2l); 91212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu(dma_hw_desc->sg3l); 91312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 91412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 91512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return xor_hw_desc->cbtal; 91612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 91712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 91812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 91912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 92012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 92112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_get_src_num - extract the number of source addresses from 92212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the descriptor 92312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 92412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 ppc440spe_desc_get_src_num(struct ppc440spe_adma_desc_slot *desc, 92512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 92612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 92712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 92812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 92912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 93012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 93112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 93212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 93312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 93412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 93512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (dma_hw_desc->opc) { 93612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_NO_OP: 93712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DFILL128: 93812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 93912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DCHECK128: 94012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 94112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MV_SG1_SG2: 94212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MULTICAST: 94312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 94412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Only for RXOR operations we have more than 94512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * one source 94612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 94712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (le32_to_cpu(dma_hw_desc->sg1u) & 94812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_WIN_MSK) { 94912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR op, there are 2 or 3 sources */ 95012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (((le32_to_cpu(dma_hw_desc->sg1u) >> 95112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF) & 95212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_MSK) == DMA_RXOR12) { 95312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR 1-2 */ 95412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 2; 95512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 95612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR 1-2-3/1-2-4/1-2-5 */ 95712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 3; 95812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 95912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 96012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 96112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 96212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: unknown OPC 0x%02x\n", 96312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dma_hw_desc->opc); 96412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 96512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 96612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 96712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* up to 16 sources */ 96812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 96912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return xor_hw_desc->cbc & XOR_CDCR_OAC_MSK; 97012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 97112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 97212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 97312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 97412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 97512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 97612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 97712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_get_dst_num - get the number of destination addresses in 97812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * this descriptor 97912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 98012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 ppc440spe_desc_get_dst_num(struct ppc440spe_adma_desc_slot *desc, 98112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 98212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 98312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 98412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 98512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 98612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 98712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 98812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* May be 1 or 2 destinations */ 98912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 99012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (dma_hw_desc->opc) { 99112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_NO_OP: 99212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DCHECK128: 99312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 99412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MV_SG1_SG2: 99512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DFILL128: 99612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 99712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MULTICAST: 99812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (desc->dst_cnt == 2) 99912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 2; 100012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 100112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 100212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 100312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: unknown OPC 0x%02x\n", 100412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dma_hw_desc->opc); 100512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 100612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 100712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 100812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Always only 1 destination */ 100912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 101012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 101112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 101212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 101312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 101412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 101512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 101612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 101712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_get_link - get the address of the descriptor that 101812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * follows this one 101912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 102012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc, 102112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 102212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 102312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!desc->hw_next) 102412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 102512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 102612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return desc->hw_next->phys; 102712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 102812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 102912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 103012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_is_aligned - check alignment 103112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 103212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline int ppc440spe_desc_is_aligned( 103312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, int num_slots) 103412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 103512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return (desc->idx & (num_slots - 1)) ? 0 : 1; 103612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 103712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 103812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 103912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_xor_slot_count - get the number of slots necessary for 104012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * XOR operation 104112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 104212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt, 104312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int *slots_per_op) 104412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 104512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 104612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 104712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* each XOR descriptor provides up to 16 source operands */ 104812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS; 104912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 105012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT)) 105112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return slot_cnt; 105212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 105312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: len %d > max %d !!\n", 105412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 105512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 105612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return slot_cnt; 105712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 105812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 105912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 106012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for 106112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * DMA2 PQ operation 106212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 106312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs, 106412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, size_t len) 106512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 106612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin signed long long order = 0; 106712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int state = 0; 106812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int addr_count = 0; 106912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 107012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 1; i < src_cnt; i++) { 107112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t cur_addr = srcs[i]; 107212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t old_addr = srcs[i-1]; 107312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (state) { 107412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 0: 107512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cur_addr == old_addr + len) { 107612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct RXOR */ 107712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 1; 107812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 1; 107912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 108012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 108112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (old_addr == cur_addr + len) { 108212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse RXOR */ 108312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = -1; 108412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 1; 108512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 108612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 108712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 108812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 3; 108912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 109012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 109112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 1: 109212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-2 || (order == -1 109312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin && cur_addr != old_addr - len)) { 109412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 109512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 109612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 109712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cur_addr == old_addr + len*order) { 109812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 2; 109912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 110012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 110112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cur_addr == old_addr + 2*len) { 110212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 2; 110312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 110412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 110512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cur_addr == old_addr + 3*len) { 110612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 2; 110712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 110812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 110912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 111012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 111112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 111212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 111312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 111412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 111512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 2: 111612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 111712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 111812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 111912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 112012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 112112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (state == 3) 112212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 112312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 112412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt <= 1 || (state != 1 && state != 2)) { 112512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n", 112612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, src_cnt, state, addr_count, order); 112712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 112812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("\t[%d] 0x%llx \n", i, srcs[i]); 112912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 113012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 113112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 113212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS; 113312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 113412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 113512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 113612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/****************************************************************************** 113712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ADMA channel low-level routines 113812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ******************************************************************************/ 113912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 114012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 114112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan); 114212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan); 114312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 114412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 114512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine 114612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 114712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_device_clear_eot_status( 114812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 114912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 115012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_regs *dma_reg; 115112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 115212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 *p = chan->device->dma_desc_pool_virt; 115312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *cdb; 115412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 rv, i; 115512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 115612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 115712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 115812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 115912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* read FIFO to ack */ 116012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_reg = chan->device->dma_reg; 116112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while ((rv = ioread32(&dma_reg->csfpl))) { 116212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i = rv & DMA_CDB_ADDR_MSK; 116312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb = (struct dma_cdb *)&p[i - 116412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (u32)chan->device->dma_desc_pool]; 116512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 116612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Clear opcode to ack. This is necessary for 116712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ZeroSum operations only 116812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 116912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb->opc = 0; 117012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 117112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_RXOR_RUN, 117212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &ppc440spe_rxor_state)) { 117312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* probably this is a completed RXOR op, 117412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * get pointer to CDB using the fact that 117512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * physical and virtual addresses of CDB 117612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * in pools have the same offsets 117712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 117812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (le32_to_cpu(cdb->sg1u) & 117912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE) { 118012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is a RXOR */ 118112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_RXOR_RUN, 118212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &ppc440spe_rxor_state); 118312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 118412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 118512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 118612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rv & DMA_CDB_STATUS_MSK) { 118712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* ZeroSum check failed 118812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 118912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 119012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t phys = rv & ~DMA_CDB_MSK; 119112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 119212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 119312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Update the status of corresponding 119412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor. 119512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 119612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &chan->chain, 119712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 119812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->phys == phys) 119912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 120012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 120112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 120212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * if cannot find the corresponding 120312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * slot it's a bug 120412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 120512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(&iter->chain_node == &chan->chain); 120612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 120712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->xor_check_result) { 120812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_PCHECK, 120912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags)) { 121012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *iter->xor_check_result |= 121112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin SUM_CHECK_P_RESULT; 121212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 121312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_QCHECK, 121412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags)) { 121512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *iter->xor_check_result |= 121612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin SUM_CHECK_Q_RESULT; 121712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 121812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 121912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 122012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 122112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 122212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 122312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rv = ioread32(&dma_reg->dsts); 122412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rv) { 122512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("DMA%d err status: 0x%x\n", 122612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id, rv); 122712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* write back to clear */ 122812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(rv, &dma_reg->dsts); 122912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 123012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 123112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 123212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reset status bits to ack */ 123312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 123412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rv = ioread32be(&xor_reg->sr); 123512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(rv, &xor_reg->sr); 123612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 123712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) { 123812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rv & XOR_IE_RPTIE_BIT) { 123912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Read PLB Timeout Error. 124012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Try to resubmit the CB 124112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 124212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 val = ioread32be(&xor_reg->ccbalr); 124312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 124412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(val, &xor_reg->cblalr); 124512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 124612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin val = ioread32be(&xor_reg->crsr); 124712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(val | XOR_CRSR_XAE_BIT, 124812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &xor_reg->crsr); 124912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 125012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("XOR ERR 0x%x status\n", rv); 125112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 125212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 125312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 125412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* if the XORcore is idle, but there are unprocessed CBs 125512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * then refetch the s/w chain here 125612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 125712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) && 125812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin do_xor_refetch) 125912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_append(chan); 126012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 126112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 126212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 126312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 126412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 126512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_is_busy - get the channel status 126612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 126712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan) 126812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 126912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_regs *dma_reg; 127012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 127112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int busy = 0; 127212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 127312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 127412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 127512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 127612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_reg = chan->device->dma_reg; 127712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* if command FIFO's head and tail pointers are equal and 127812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * status tail is the same as command, then channel is free 127912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 128012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) || 128112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp)) 128212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin busy = 1; 128312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 128412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 128512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* use the special status bit for the XORcore 128612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 128712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 128812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0; 128912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 129012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 129112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 129212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return busy; 129312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 129412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 129512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 129612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_set_first_xor_descriptor - init XORcore chain 129712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 129812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_set_first_xor_descriptor( 129912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 130012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *next_desc) 130112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 130212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg = chan->device->xor_reg; 130312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 130412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) 130512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_INFO "%s: Warn: XORcore is running " 130612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "when try to set the first CDB!\n", 130712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 130812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 130912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_submit = xor_last_linked = next_desc; 131012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 131112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr); 131212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 131312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(next_desc->phys, &xor_reg->cblalr); 131412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(0, &xor_reg->cblahr); 131512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT, 131612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &xor_reg->cbcr); 131712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 131812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->hw_chain_inited = 1; 131912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 132012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 132112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 132212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO. 132312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * called with irqs disabled 132412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 132512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan, 132612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc) 132712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 132812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 pcdb; 132912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_regs *dma_reg = chan->device->dma_reg; 133012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 133112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pcdb = desc->phys; 133212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_INT, &desc->flags)) 133312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pcdb |= DMA_CDB_NO_INT; 133412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 133512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan_last_sub[chan->device->id] = desc; 133612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 133712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(print_cb(chan, desc->hw_desc)); 133812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 133912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(pcdb, &dma_reg->cpfpl); 134012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 134112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 134212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 134312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_append - update the h/w chain in the channel 134412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 134512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan) 134612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 134712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 134812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 134912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb; 135012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 cur_desc; 135112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long flags; 135212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 135312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin local_irq_save(flags); 135412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 135512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 135612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 135712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 135812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cur_desc = ppc440spe_chan_get_current_descriptor(chan); 135912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 136012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(cur_desc)) { 136112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = chan_last_sub[chan->device->id]; 136212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!iter); 136312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 136412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* first peer */ 136512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = chan_first_cdb[chan->device->id]; 136612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!iter); 136712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_dma_put_desc(chan, iter); 136812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->hw_chain_inited = 1; 136912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 137012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 137112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* is there something new to append */ 137212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!iter->hw_next) 137312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 137412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 137512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* flush descriptors from the s/w queue to fifo */ 137612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_continue(iter, &chan->chain, chain_node) { 137712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_dma_put_desc(chan, iter); 137812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!iter->hw_next) 137912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 138012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 138112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 138212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 138312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* update h/w links and refetch */ 138412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!xor_last_submit->hw_next) 138512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 138612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 138712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 138812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* the last linked CDB has to generate an interrupt 138912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * that we'd be able to append the next lists to h/w 139012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * regardless of the XOR engine state at the moment of 139112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * appending of these next lists 139212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 139312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb = xor_last_linked->hw_desc; 139412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->cbc |= XOR_CBCR_CBCE_BIT; 139512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 139612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) { 139712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* XORcore is idle. Refetch now */ 139812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin do_xor_refetch = 0; 139912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_xor_set_link(xor_last_submit, 140012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_submit->hw_next); 140112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 140212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(print_cb_list(chan, 140312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_submit->hw_next)); 140412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 140512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_submit = xor_last_linked; 140612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(ioread32be(&xor_reg->crsr) | 140712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT, 140812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &xor_reg->crsr); 140912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 141012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* XORcore is running. Refetch later in the handler */ 141112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin do_xor_refetch = 1; 141212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 141312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 141412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 141512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 141612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 141712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin local_irq_restore(flags); 141812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 141912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 142012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 142112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor 142212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 142312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 142412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan) 142512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 142612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_regs *dma_reg; 142712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 142812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 142912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!chan->hw_chain_inited)) 143012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* h/w descriptor chain is not initialized yet */ 143112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 143212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 143312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 143412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 143512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 143612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_reg = chan->device->dma_reg; 143712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK); 143812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 143912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 144012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ioread32be(&xor_reg->ccbalr); 144112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 144212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 144312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 144412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 144512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 144612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_run - enable the channel 144712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 144812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan) 144912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 145012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 145112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 145212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 145312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 145412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 145512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMAs are always enabled, do nothing */ 145612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 145712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 145812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* drain write buffer */ 145912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 146012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 146112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* fetch descriptor pointed to in <link> */ 146212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT, 146312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &xor_reg->crsr); 146412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 146512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 146612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 146712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 146812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/****************************************************************************** 146912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ADMA device level 147012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ******************************************************************************/ 147112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 147212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan); 147312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan); 147412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 147512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic dma_cookie_t 147612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx); 147712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 147812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx, 147912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index); 148012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 148112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx, 148212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index); 148312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 148412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 148512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx, 148612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *paddr, unsigned long flags); 148712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 148812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx, 148912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index); 149012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 149112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx, 149212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned char mult, int index, int dst_pos); 149312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 149412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx, 149512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t paddr, dma_addr_t qaddr); 149612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 149712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct page *ppc440spe_rxor_srcs[32]; 149812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 149912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 150012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_can_rxor - check if the operands may be processed with RXOR 150112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 150212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len) 150312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 150412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i, order = 0, state = 0; 150512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int idx = 0; 150612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 150712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!(src_cnt > 1))) 150812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 150912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 151012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs)); 151112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 151212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Skip holes in the source list before checking */ 151312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) { 151412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!srcs[i]) 151512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 151612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_srcs[idx++] = srcs[i]; 151712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 151812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt = idx; 151912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 152012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 1; i < src_cnt; i++) { 152112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin char *cur_addr = page_address(ppc440spe_rxor_srcs[i]); 152212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]); 152312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 152412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (state) { 152512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 0: 152612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cur_addr == old_addr + len) { 152712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct RXOR */ 152812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 1; 152912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 1; 153012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (old_addr == cur_addr + len) { 153112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse RXOR */ 153212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = -1; 153312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 1; 153412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 153512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out; 153612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 153712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 1: 153812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((i == src_cnt - 2) || 153912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (order == -1 && cur_addr != old_addr - len)) { 154012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 154112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 154212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if ((cur_addr == old_addr + len * order) || 154312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (cur_addr == old_addr + 2 * len) || 154412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (cur_addr == old_addr + 3 * len)) { 154512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 2; 154612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 154712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 154812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 154912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 155012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 155112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 2: 155212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 155312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 155412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 155512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 155612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 155712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 155812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout: 155912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (state == 1 || state == 2) 156012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 156112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 156212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 156312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 156412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 156512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 156612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_device_estimate - estimate the efficiency of processing 156712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the operation given on this channel. It's assumed that 'chan' is 156812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * capable to process 'cap' type of operation. 156912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @chan: channel to use 157012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @cap: type of transaction 157112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @dst_lst: array of destination pointers 157212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @dst_cnt: number of destination operands 157312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @src_lst: array of source pointers 157412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @src_cnt: number of source operands 157512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @src_sz: size of each source operand 157612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 157712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_estimate(struct dma_chan *chan, 157812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt, 157912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct page **src_lst, int src_cnt, size_t src_sz) 158012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 158112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int ef = 1; 158212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 158312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cap == DMA_PQ || cap == DMA_PQ_VAL) { 158412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* If RAID-6 capabilities were not activated don't try 158512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * to use them 158612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 158712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!ppc440spe_r6_enabled)) 158812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -1; 158912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 159012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* In the current implementation of ppc440spe ADMA driver it 159112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * makes sense to pick out only pq case, because it may be 159212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * processed: 159312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (1) either using Biskup method on DMA2; 159412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (2) or on DMA0/1. 159512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Thus we give a favour to (1) if the sources are suitable; 159612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * else let it be processed on one of the DMA0/1 engines. 159712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * In the sum_product case where destination is also the 159812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * source process it on DMA0/1 only. 159912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 160012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) { 160112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 160212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1]) 160312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ef = 0; /* sum_product case, process on DMA0/1 */ 160412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz)) 160512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ef = 3; /* override (DMA0/1 + idle) */ 160612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 160712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ef = 0; /* can't process on DMA2 if !rxor */ 160812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 160912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 161012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* channel idleness increases the priority */ 161112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(ef) && 161212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan))) 161312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ef++; 161412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 161512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ef; 161612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 161712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 161812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstruct dma_chan * 161912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap, 162012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct page **dst_lst, int dst_cnt, struct page **src_lst, 162112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, size_t src_sz) 162212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 162312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *best_chan = NULL; 162412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc_dma_chan_ref *ref; 162512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int best_rank = -1; 162612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 162712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!src_sz)) 162812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 162912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_sz > PAGE_SIZE) { 163012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 163112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * should a user of the api ever pass > PAGE_SIZE requests 163212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * we sort out cases where temporary page-sized buffers 163312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * are used. 163412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 163512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (cap) { 163612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_PQ: 163712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt == 1 && dst_lst[1] == src_lst[0]) 163812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 163912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt == 2 && dst_lst[1] == src_lst[1]) 164012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 164112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 164212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_PQ_VAL: 164312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_XOR_VAL: 164412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 164512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 164612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 164712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 164812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 164912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 165012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) { 165112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(cap, ref->chan->device->cap_mask)) { 165212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int rank; 165312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 165412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst, 165512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst_cnt, src_lst, src_cnt, src_sz); 165612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rank > best_rank) { 165712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin best_rank = rank; 165812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin best_chan = ref->chan; 165912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 166012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 166112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 166212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 166312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return best_chan; 166412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 166512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinEXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel); 166612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 166712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 166812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_get_group_entry - get group entry with index idx 166912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @tdesc: is the last allocated slot in the group. 167012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 167112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot * 167212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx) 167312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 167412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter = tdesc->group_head; 167512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i = 0; 167612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 167712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) { 167812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n", 167912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt); 168012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 168112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 168212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 168312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &tdesc->group_list, chain_node) { 168412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i++ == entry_idx) 168512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 168612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 168712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return iter; 168812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 168912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 169012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 169112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_free_slots - flags descriptor slots for reuse 169212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @slot: Slot to free 169312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Caller must hold &ppc440spe_chan->lock while calling this function 169412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 169512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot, 169612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 169712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 169812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int stride = slot->slots_per_op; 169912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 170012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (stride--) { 170112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->slots_per_op = 0; 170212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot = list_entry(slot->slot_node.next, 170312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 170412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node); 170512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 170612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 170712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 170812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_unmap(struct ppc440spe_adma_chan *chan, 170912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc) 171012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 171112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 src_cnt, dst_cnt; 171212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr; 171312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 171412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 171512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * get the number of sources & destination 171612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * included in this descriptor and unmap 171712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * them all 171812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 171912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt = ppc440spe_desc_get_src_num(desc, chan); 172012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst_cnt = ppc440spe_desc_get_dst_num(desc, chan); 172112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 172212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* unmap destinations */ 172312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(desc->async_tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) { 172412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (dst_cnt--) { 172512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr = ppc440spe_desc_get_dest_addr( 172612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, chan, dst_cnt); 172712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(chan->device->dev, 172812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr, desc->unmap_len, 172912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_FROM_DEVICE); 173012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 173112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 173212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 173312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* unmap sources */ 173412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(desc->async_tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) { 173512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (src_cnt--) { 173612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr = ppc440spe_desc_get_src_addr( 173712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, chan, src_cnt); 173812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(chan->device->dev, 173912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr, desc->unmap_len, 174012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_TO_DEVICE); 174112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 174212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 174312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 174412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 174512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 174612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_run_tx_complete_actions - call functions to be called 174712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * upon completion 174812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 174912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic dma_cookie_t ppc440spe_adma_run_tx_complete_actions( 175012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 175112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 175212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t cookie) 175312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 175412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 175512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 175612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(desc->async_tx.cookie < 0); 175712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (desc->async_tx.cookie > 0) { 175812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = desc->async_tx.cookie; 175912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->async_tx.cookie = 0; 176012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 176112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* call the callback (must not sleep or submit new 176212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * operations to this channel) 176312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 176412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (desc->async_tx.callback) 176512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->async_tx.callback( 176612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->async_tx.callback_param); 176712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 1768d38a8c622a1b382336c3e152c6caf4e11d1f1b2aDan Williams dma_descriptor_unmap(&desc->async_tx); 176912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* unmap dma addresses 177012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (unmap_single vs unmap_page?) 177112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 177212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * actually, ppc's dma_unmap_page() functions are empty, so 177312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the following code is just for the sake of completeness 177412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 177512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan && chan->needs_unmap && desc->group_head && 177612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->unmap_len) { 177712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *unmap = 177812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->group_head; 177912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* assume 1 slot per op always */ 178012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 slot_count = unmap->slot_cnt; 178112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 178212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Run through the group list and unmap addresses */ 178312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < slot_count; i++) { 178412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!unmap); 178512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_unmap(chan, unmap); 178612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unmap = unmap->hw_next; 178712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 178812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 178912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 179012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 179112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* run dependent operations */ 179212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_run_dependencies(&desc->async_tx); 179312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 179412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return cookie; 179512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 179612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 179712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 179812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set) 179912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 180012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc, 180112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 180212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 180312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* the client is allowed to attach dependent operations 180412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * until 'ack' is set 180512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 180612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!async_tx_test_ack(&desc->async_tx)) 180712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 180812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 180912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* leave the last descriptor in the chain 181012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * so we can append to it 181112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 181212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (list_is_last(&desc->chain_node, &chan->chain) || 181312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->phys == ppc440spe_chan_get_current_descriptor(chan)) 181412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 181512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 181612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan->device->id != PPC440SPE_XOR_ID) { 181712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* our DMA interrupt handler clears opc field of 181812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * each processed descriptor. For all types of 181912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * operations except for ZeroSum we do not actually 182012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * need ack from the interrupt handler. ZeroSum is a 182112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * special case since the result of this operation 182212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * is available from the handler only, so if we see 182312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * such type of descriptor (which is unprocessed yet) 182412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * then leave it in chain. 182512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 182612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *cdb = desc->hw_desc; 182712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cdb->opc == DMA_CDB_OPC_DCHECK128) 182812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 182912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 183012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 183112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n", 183212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->phys, desc->idx, desc->slots_per_op); 183312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 183412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&desc->chain_node); 183512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_free_slots(desc, chan); 183612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 183712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 183812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 183912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 184012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine 184112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * which runs through the channel CDBs list until reach the descriptor 184212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * currently processed. When routine determines that all CDBs of group 184312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * are completed then corresponding callbacks (if any) are called and slots 184412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * are freed. 184512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 184612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan) 184712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 184812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL; 184912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t cookie = 0; 185012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 current_desc = ppc440spe_chan_get_current_descriptor(chan); 185112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int busy = ppc440spe_chan_is_busy(chan); 185212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int seen_current = 0, slot_cnt = 0, slots_per_op = 0; 185312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 185412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n", 185512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id, __func__); 185612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 185712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!current_desc) { 185812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* There were no transactions yet, so 185912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * nothing to clean 186012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 186112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return; 186212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 186312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 186412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* free completed slots from the chain starting with 186512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the oldest descriptor 186612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 186712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(iter, _iter, &chan->chain, 186812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 186912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d " 187012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "busy: %d this_desc: %#llx next_desc: %#x " 187112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "cur: %#x ack: %d\n", 187212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->async_tx.cookie, iter->idx, busy, iter->phys, 187312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_get_link(iter, chan), current_desc, 187412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin async_tx_test_ack(&iter->async_tx)); 187512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prefetch(_iter); 187612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prefetch(&_iter->async_tx); 187712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 187812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do not advance past the current descriptor loaded into the 187912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * hardware channel,subsequent descriptors are either in process 188012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * or have not been submitted 188112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 188212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (seen_current) 188312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 188412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 188512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* stop the search if we reach the current descriptor and the 188612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * channel is busy, or if it appears that the current descriptor 188712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * needs to be re-read (i.e. has been appended to) 188812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 188912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->phys == current_desc) { 189012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(seen_current++); 189112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (busy || ppc440spe_desc_get_link(iter, chan)) { 189212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* not all descriptors of the group have 189312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * been completed; exit. 189412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 189512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 189612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 189712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 189812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 189912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* detect the start of a group transaction */ 190012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!slot_cnt && !slots_per_op) { 190112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = iter->slot_cnt; 190212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = iter->slots_per_op; 190312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slot_cnt <= slots_per_op) { 190412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = 0; 190512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = 0; 190612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 190712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 190812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 190912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slot_cnt) { 191012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!group_start) 191112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = iter; 191212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt -= slots_per_op; 191312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 191412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 191512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* all the members of a group are complete */ 191612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slots_per_op != 0 && slot_cnt == 0) { 191712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter; 191812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int end_of_chain = 0; 191912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 192012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* clean up the group */ 192112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = group_start->slot_cnt; 192212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin grp_iter = group_start; 192312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe_from(grp_iter, _grp_iter, 192412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &chan->chain, chain_node) { 192512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 192612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = ppc440spe_adma_run_tx_complete_actions( 192712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin grp_iter, chan, cookie); 192812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 192912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt -= slots_per_op; 193012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin end_of_chain = ppc440spe_adma_clean_slot( 193112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin grp_iter, chan); 193212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (end_of_chain && slot_cnt) { 193312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Should wait for ZeroSum completion */ 193412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cookie > 0) 19354d4e58de32a192fea65ab84509d17d199bd291c8Russell King - ARM Linux chan->common.completed_cookie = cookie; 193612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return; 193712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 193812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 193912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slot_cnt == 0 || end_of_chain) 194012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 194112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 194212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 194312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* the group should be complete at this point */ 194412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(slot_cnt); 194512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 194612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = 0; 194712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = NULL; 194812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (end_of_chain) 194912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 195012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 195112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 195212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (slots_per_op) /* wait for group completion */ 195312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 195412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 195512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan, 195612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie); 195712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 195812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_adma_clean_slot(iter, chan)) 195912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 196012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 196112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 196212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!seen_current); 196312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 196412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cookie > 0) { 19654d4e58de32a192fea65ab84509d17d199bd291c8Russell King - ARM Linux chan->common.completed_cookie = cookie; 196612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\tcompleted cookie %d\n", cookie); 196712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 196812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 196912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 197012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 197112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 197212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_tasklet - clean up watch-dog initiator 197312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 197412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_tasklet(unsigned long data) 197512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 197612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan = (struct ppc440spe_adma_chan *) data; 197712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 197812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING); 197912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __ppc440spe_adma_slot_cleanup(chan); 198012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock(&chan->lock); 198112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 198212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 198312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 198412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_slot_cleanup - clean up scheduled initiator 198512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 198612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan) 198712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 198812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&chan->lock); 198912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __ppc440spe_adma_slot_cleanup(chan); 199012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 199112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 199212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 199312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 199412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_alloc_slots - allocate free slots (if any) 199512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 199612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots( 199712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, int num_slots, 199812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slots_per_op) 199912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 200012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter = NULL, *_iter; 200112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *alloc_start = NULL; 200212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct list_head chain = LIST_HEAD_INIT(chain); 200312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slots_found, retry = 0; 200412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 200512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 200612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!num_slots || !slots_per_op); 200712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* start search from the last allocated descrtiptor 200812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * if a contiguous allocation can not be found start searching 200912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * from the beginning of the list 201012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 201112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinretry: 201212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_found = 0; 201312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (retry == 0) 201412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = chan->last_used; 201512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 201612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(&chan->all_slots, 201712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 201812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node); 201912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots, 202012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node) { 202112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prefetch(_iter); 202212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prefetch(&_iter->async_tx); 202312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->slots_per_op) { 202412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_found = 0; 202512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 202612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 202712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 202812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* start the allocation if the slot is correctly aligned */ 202912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!slots_found++) 203012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin alloc_start = iter; 203112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 203212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slots_found == num_slots) { 203312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *alloc_tail = NULL; 203412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *last_used = NULL; 203512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 203612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = alloc_start; 203712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (num_slots) { 203812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 203912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* pre-ack all but the last descriptor */ 204012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (num_slots != slots_per_op) 204112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin async_tx_ack(&iter->async_tx); 204212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 204312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_add_tail(&iter->chain_node, &chain); 204412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin alloc_tail = iter; 204512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->async_tx.cookie = 0; 204612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 204712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->flags = 0; 204812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->slot_cnt = num_slots; 204912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->xor_check_result = NULL; 205012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < slots_per_op; i++) { 205112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->slots_per_op = slots_per_op - i; 205212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin last_used = iter; 205312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->slot_node.next, 205412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 205512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node); 205612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 205712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin num_slots -= slots_per_op; 205812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 205912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin alloc_tail->group_head = alloc_start; 206012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin alloc_tail->async_tx.cookie = -EBUSY; 206112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_splice(&chain, &alloc_tail->group_list); 206212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->last_used = last_used; 206312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return alloc_tail; 206412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 206512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 206612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!retry++) 206712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto retry; 206812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 206912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* try to free some slots if the allocation fails */ 207012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_schedule(&chan->irq_tasklet); 207112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 207212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 207312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 207412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 207512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_alloc_chan_resources - allocate pools for CDB slots 207612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 207712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan) 207812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 207912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 208012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *slot = NULL; 208112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin char *hw_desc; 208212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i, db_sz; 208312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int init; 208412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 208512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 208612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin init = ppc440spe_chan->slots_allocated ? 0 : 1; 208712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->chan_id = ppc440spe_chan->device->id; 208812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 208912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Allocate descriptor slots */ 209012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i = ppc440spe_chan->slots_allocated; 209112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID) 209212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin db_sz = sizeof(struct dma_cdb); 209312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 209412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin db_sz = sizeof(struct xor_cb); 209512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 209612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) { 209712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot), 209812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin GFP_KERNEL); 209912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!slot) { 210012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_INFO "SPE ADMA Channel only initialized" 210112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " %d descriptor slots", i--); 210212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 210312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 210412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 210512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt; 210612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->hw_desc = (void *) &hw_desc[i * db_sz]; 210712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_async_tx_descriptor_init(&slot->async_tx, chan); 210812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->async_tx.tx_submit = ppc440spe_adma_tx_submit; 210912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&slot->chain_node); 211012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&slot->slot_node); 211112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&slot->group_list); 211212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz; 211312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->idx = i; 211412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 211512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 211612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->slots_allocated++; 211712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots); 211812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 211912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 212012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 212112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i && !ppc440spe_chan->last_used) { 212212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->last_used = 212312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_entry(ppc440spe_chan->all_slots.next, 212412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 212512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node); 212612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 212712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 212812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 212912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: allocated %d descriptor slots\n", 213012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, i); 213112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 213212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* initialize the channel and the chain with a null operation */ 213312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (init) { 213412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (ppc440spe_chan->device->id) { 213512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 213612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 213712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->hw_chain_inited = 0; 213812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Use WXOR for self-testing */ 213912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!ppc440spe_r6_tchan) 214012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_r6_tchan = ppc440spe_chan; 214112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 214212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 214312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_start_null_xor(ppc440spe_chan); 214412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 214512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 214612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 214712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 214812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->needs_unmap = 1; 214912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 215012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 215112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return (i > 0) ? i : -ENOMEM; 215212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 215312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 215412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 215512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_rxor_set_region_data - 215612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 215712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc, 215812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 xor_arg_no, u32 mask) 215912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 216012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 216112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 216212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->ops[xor_arg_no].h |= mask; 216312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 216412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 216512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 216612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_rxor_set_src - 216712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 216812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc, 216912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 xor_arg_no, dma_addr_t addr) 217012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 217112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 217212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 217312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE; 217412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->ops[xor_arg_no].l = addr; 217512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 217612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 217712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 217812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_rxor_set_mult - 217912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 218012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc, 218112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 xor_arg_no, u8 idx, u8 mult) 218212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 218312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 218412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 218512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8); 218612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 218712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 218812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 218912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold 219012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * has been achieved 219112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 219212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan) 219312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 219412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n", 219512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id, chan->pending); 219612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 219712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) { 219812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->pending = 0; 219912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_append(chan); 220012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 220112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 220212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 220312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 220412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_tx_submit - submit new descriptor group to the channel 220512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (it's not necessary that descriptors will be submitted to the h/w 220612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * chains too right now) 220712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 220812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx) 220912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 221012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc; 221112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan); 221212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail; 221312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 221412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slots_per_op; 221512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t cookie; 221612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 221712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = tx_to_ppc440spe_adma_slot(tx); 221812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 221912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 222012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = group_start->slot_cnt; 222112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = group_start->slots_per_op; 222212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 222312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&chan->lock); 2224884485e1f12dcd39390f042e772cdbefc9ebb750Russell King - ARM Linux cookie = dma_cookie_assign(tx); 222512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 222612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(list_empty(&chan->chain))) { 222712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* first peer */ 222812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_splice_init(&sw_desc->group_list, &chan->chain); 222912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan_first_cdb[chan->device->id] = group_start; 223012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 223112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* isn't first peer, bind CDBs to chain */ 223212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin old_chain_tail = list_entry(chan->chain.prev, 223312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 223412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 223512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_splice_init(&sw_desc->group_list, 223612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &old_chain_tail->chain_node); 223712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* fix up the hardware chain */ 223812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_link(chan, old_chain_tail, group_start); 223912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 224012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 224112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* increment the pending count by the number of operations */ 224212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->pending += slot_cnt / slots_per_op; 224312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_check_threshold(chan); 224412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 224512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 224612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, 224712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n", 224812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id, __func__, 224912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.cookie, sw_desc->idx, sw_desc); 225012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 225112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return cookie; 225212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 225312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 225412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 225512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation 225612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 225712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt( 225812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, unsigned long flags) 225912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 226012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 226112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 226212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op; 226312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 226412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 226512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 226612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 226712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id, 226812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 226912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 227012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 227112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = slots_per_op = 1; 227212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 227312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op); 227412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 227512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 227612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan); 227712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start->unmap_len = 0; 227812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 227912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 228012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 228112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 228212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 228312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 228412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 228512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 228612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation 228712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 228812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy( 228912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t dma_dest, 229012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t dma_src, size_t len, unsigned long flags) 229112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 229212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 229312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 229412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op; 229512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 229612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 229712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 229812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!len)) 229912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 230012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 2301427cdf19b97e509e21e5d347e18d8b0b34723dfcColy Li BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT); 230212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 230312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 230412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 230512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 230612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s len: %u int_en %d\n", 230712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, __func__, len, 230812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags & DMA_PREP_INTERRUPT ? 1 : 0); 230912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = slots_per_op = 1; 231012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 231112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op); 231212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 231312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 231412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_memcpy(group_start, flags); 231512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_set_dest(group_start, dma_dest, 0); 231612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0); 231712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len); 231812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->unmap_len = len; 231912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 232012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 232112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 232212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 232312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 232412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 232512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 232612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 232712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation 232812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 232912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor( 233012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t dma_dest, 233112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dma_src, u32 src_cnt, size_t len, 233212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long flags) 233312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 233412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 233512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 233612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op; 233712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 233812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 233912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 234012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id, 234112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_dest, dma_src, src_cnt)); 234212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!len)) 234312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 2344427cdf19b97e509e21e5d347e18d8b0b34723dfcColy Li BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 234512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 234612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 234712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n", 234812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, __func__, src_cnt, len, 234912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags & DMA_PREP_INTERRUPT ? 1 : 0); 235012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 235112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 235212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op); 235312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 235412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op); 235512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 235612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 235712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_xor(group_start, src_cnt, flags); 235812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_set_dest(group_start, dma_dest, 0); 235912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (src_cnt--) 236012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_memcpy_xor_set_src(group_start, 236112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_src[src_cnt], src_cnt); 236212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len); 236312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->unmap_len = len; 236412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 236512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 236612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 236712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 236812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 236912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 237012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 237112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline void 237212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc, 237312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt); 237412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor); 237512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 237612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 237712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_init_dma2rxor_slot - 237812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 237912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_init_dma2rxor_slot( 238012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 238112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *src, int src_cnt) 238212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 238312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 238412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 238512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* initialize CDB */ 238612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) { 238712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i, 238812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt, (u32)src[i]); 238912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 239012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 239112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 239212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 239312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma01_prep_mult - 239412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for Q operation where destination is also the source 239512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 239612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult( 239712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan, 239812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 239912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf, size_t len, unsigned long flags) 240012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 240112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL; 240212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 240312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 240412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 240512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_WXOR, &op); 240612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = 2; 240712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 240812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 240912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 241012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* use WXOR, each descriptor occupies one slot */ 241112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 241212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 241312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 241412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 241512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 241612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 241712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 241812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bits(op, &sw_desc->flags); 241912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->src_cnt = src_cnt; 242012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt = dst_cnt; 242112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* First descriptor, zero data in the destination and copy it 242212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * to q page using MULTICAST transfer. 242312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 242412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&sw_desc->group_list, 242512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 242612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 242712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 242812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 242912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 243012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 243112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 243212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 243312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 243412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MULTICAST; 243512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 243612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 243712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, dst[0], 0); 243812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1); 243912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 244012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src[0]); 244112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 244212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 244312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 244412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 244512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Second descriptor, multiply data from the q page 244612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * and store the result in real destination. 244712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 244812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 244912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 245012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 245112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 245212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 245312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 245412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 245512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 245612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 245712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 245812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 245912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 246012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 246112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB, dst[1]); 246212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 246312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, dst[0], 0); 246412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 246512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 246612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1, scf[0]); 246712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 246812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 246912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 247012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 247112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 247212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 247312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 247412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc; 247512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 247612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 247712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 247812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma01_prep_sum_product - 247912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also 248012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the source. 248112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 248212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product( 248312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan, 248412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dst, dma_addr_t *src, int src_cnt, 248512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf, size_t len, unsigned long flags) 248612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 248712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL; 248812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 248912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 249012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 249112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_WXOR, &op); 249212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = 3; 249312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 249412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 249512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 249612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR, each descriptor occupies one slot */ 249712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 249812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 249912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 250012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 250112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 250212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 250312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 250412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bits(op, &sw_desc->flags); 250512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->src_cnt = src_cnt; 250612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt = 1; 250712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 1st descriptor, src[1] data to q page and zero destination */ 250812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&sw_desc->group_list, 250912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 251012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 251112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 251212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 251312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 251412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 251512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 251612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 251712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MULTICAST; 251812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 251912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 252012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *dst, 0); 252112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 0, 252212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->qdest, 1); 252312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 252412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src[1]); 252512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 252612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 252712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 252812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 2nd descriptor, multiply src[1] data and store the 252912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * result in destination */ 253012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 253112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 253212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 253312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 253412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 253512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 253612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 253712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 253812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 253912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 254012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 254112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 254212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 254312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 254412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 254512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 254612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->qdest); 254712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 254812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *dst, 0); 254912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 255012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1, scf[1]); 255112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 255212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 255312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 255412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 255512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 3rd descriptor, multiply src[0] data and xor it 255612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * with destination 255712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 255812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 255912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 256012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 256112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 256212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 256312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 256412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 256512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 256612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 256712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 256812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 256912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 257012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 257112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src[0]); 257212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 257312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *dst, 0); 257412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 257512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1, scf[0]); 257612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 257712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 257812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 257912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 258012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 258112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 258212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 258312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc; 258412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 258512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 258612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq( 258712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan, 258812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 258912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf, size_t len, unsigned long flags) 259012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 259112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 259212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter; 259312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 259412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned char mult = 1; 259512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 259612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n", 259712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dst_cnt, src_cnt, len); 259812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* select operations WXOR/RXOR depending on the 259912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * source addresses of operators and the number 260012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * of destinations (RXOR support only Q-parity calculations) 260112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 260212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_WXOR, &op); 260312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) { 260412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* no active RXOR; 260512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * do RXOR if: 260612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - there are more than 1 source, 260712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - len is aligned on 512-byte boundary, 260812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - source addresses fit to one of 4 possible regions. 260912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 261012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt > 1 && 261112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin !(len & MQ0_CF2H_RXOR_BS_MASK) && 261212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (src[0] + len) == src[1]) { 261312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* may do RXOR R1 R2 */ 261412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR, &op); 261512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt != 2) { 261612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* may try to enhance region of RXOR */ 261712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((src[1] + len) == src[2]) { 261812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 R3 */ 261912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR123, 262012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &op); 262112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if ((src[1] + len * 2) == src[2]) { 262212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 R4 */ 262312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR124, &op); 262412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if ((src[1] + len * 3) == src[2]) { 262512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 R5 */ 262612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR125, 262712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &op); 262812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 262912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 */ 263012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR12, 263112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &op); 263212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 263312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 263412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 */ 263512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR12, &op); 263612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 263712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 263812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 263912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_RXOR, &op)) { 264012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* can not do this operation with RXOR */ 264112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_RXOR_RUN, 264212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &ppc440spe_rxor_state); 264312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 264412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* can do; set block size right now */ 264512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_rxor_block_size(len); 264612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 264712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 264812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 264912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Number of necessary slots depends on operation type selected */ 265012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_RXOR, &op)) { 265112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is a WXOR only chain. Need descriptors for each 265212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * source to GF-XOR them with WXOR, and need descriptors 265312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for each destination to zero them with WXOR 265412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 265512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = src_cnt; 265612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 265712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_P) { 265812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt++; 265912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_P, &op); 266012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 266112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_Q) { 266212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt++; 266312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_Q, &op); 266412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 266512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 266612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Need 1/2 descriptor for RXOR operation, and 266712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * need (src_cnt - (2 or 3)) for WXOR of sources 266812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * remained (if any) 266912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 267012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = dst_cnt; 267112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 267212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_P) 267312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_P, &op); 267412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_Q) 267512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_Q, &op); 267612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 267712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_RXOR12, &op)) 267812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt += src_cnt - 2; 267912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 268012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt += src_cnt - 3; 268112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 268212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Thus we have either RXOR only chain or 268312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * mixed RXOR/WXOR 268412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 268512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slot_cnt == dst_cnt) 268612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR only chain */ 268712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_WXOR, &op); 268812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 268912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 269012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 269112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* for both RXOR/WXOR each descriptor occupies one slot */ 269212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 269312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 269412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt, 269512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags, op); 269612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 269712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* setup dst/src/mult */ 269812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n", 269912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dst[0], dst[1]); 270012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_dest(sw_desc, dst, flags); 270112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (src_cnt--) { 270212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt], 270312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt); 270412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 270512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* NOTE: "Multi = 0 is equivalent to = 1" as it 270612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf 270712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * doesn't work for RXOR with DMA0/1! Instead, multi=0 270812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * leads to zeroing source data after RXOR. 270912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * So, for P case set-up mult=1 explicitly. 271012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 271112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 271212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult = scf[src_cnt]; 271312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src_mult(sw_desc, 271412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult, src_cnt, dst_cnt - 1); 271512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 271612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 271712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup byte count foreach slot just allocated */ 271812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 271912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, 272012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 272112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, 272212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan, len); 272312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 272412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 272512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 272612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 272712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 272812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc; 272912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 273012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 273112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq( 273212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan, 273312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 273412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf, size_t len, unsigned long flags) 273512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 273612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, descs_per_op; 273712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter; 273812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 273912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned char mult = 1; 274012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 274112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!dst_cnt); 274212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n", 274312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dst_cnt, src_cnt, len);*/ 274412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 274512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 274612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len); 274712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (descs_per_op < 0) { 274812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 274912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 275012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 275112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 275212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* depending on number of sources we have 1 or 2 RXOR chains */ 275312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = descs_per_op * dst_cnt; 275412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 275512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 275612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 275712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op = slot_cnt; 275812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 275912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 276012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt, 276112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin --op ? 0 : flags); 276212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 276312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len); 276412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 276512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 276612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_init_rxor_cursor(&(iter->rxor_cursor)); 276712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->rxor_cursor.len = len; 276812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->descs_per_op = descs_per_op; 276912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 277012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op = 0; 277112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 277212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op++; 277312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (op % descs_per_op == 0) 277412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_init_dma2rxor_slot(iter, src, 277512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt); 277612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(!list_is_last(&iter->chain_node, 277712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list))) { 277812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 277912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = 278012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_entry(iter->chain_node.next, 278112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 278212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 278312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_xor_set_link(iter, iter->hw_next); 278412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 278512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is the last descriptor. */ 278612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 278712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 278812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 278912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 279012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* fixup head descriptor */ 279112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt = dst_cnt; 279212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_P) 279312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_P, &sw_desc->flags); 279412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_Q) 279512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags); 279612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 279712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* setup dst/src/mult */ 279812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_dest(sw_desc, dst, flags); 279912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 280012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (src_cnt--) { 280112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* handle descriptors (if dst_cnt == 2) inside 280212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the ppc440spe_adma_pq_set_srcxxx() functions 280312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 280412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt], 280512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt); 280612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 280712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult = scf[src_cnt]; 280812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src_mult(sw_desc, 280912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult, src_cnt, dst_cnt - 1); 281012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 281112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 281212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 281312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_rxor_block_size(len); 281412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc; 281512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 281612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 281712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 281812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation 281912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 282012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq( 282112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, 282212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt, const unsigned char *scf, 282312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size_t len, unsigned long flags) 282412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 282512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 282612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL; 282712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int dst_cnt = 0; 282812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 282912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 283012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 283112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id, 283212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst, src, src_cnt)); 283312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!len); 2834427cdf19b97e509e21e5d347e18d8b0b34723dfcColy Li BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 283512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!src_cnt); 283612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 283712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt == 1 && dst[1] == src[0]) { 283812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t dest[2]; 283912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 284012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* dst[1] is real destination (Q) */ 284112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dest[0] = dst[1]; 284212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is the page to multicast source data to */ 284312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dest[1] = ppc440spe_chan->qdest; 284412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan, 284512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dest, 2, src, src_cnt, scf, len, flags); 284612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 284712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 284812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 284912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt == 2 && dst[1] == src[1]) { 285012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan, 285112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &dst[1], src, 2, scf, len, flags); 285212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 285312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 285412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 285512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(flags & DMA_PREP_PQ_DISABLE_P)) { 285612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!dst[0]); 285712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst_cnt++; 285812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags |= DMA_PREP_ZERO_P; 285912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 286012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 286112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(flags & DMA_PREP_PQ_DISABLE_Q)) { 286212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!dst[1]); 286312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst_cnt++; 286412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags |= DMA_PREP_ZERO_Q; 286512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 286612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 286712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!dst_cnt); 286812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 286912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 287012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n", 287112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, __func__, src_cnt, len, 287212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags & DMA_PREP_INTERRUPT ? 1 : 0); 287312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 287412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (ppc440spe_chan->device->id) { 287512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 287612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 287712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan, 287812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst, dst_cnt, src, src_cnt, scf, 287912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len, flags); 288012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 288112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 288212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 288312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan, 288412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst, dst_cnt, src, src_cnt, scf, 288512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len, flags); 288612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 288712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 288812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 288912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 289012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 289112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 289212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 289312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for 289412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * a PQ_ZERO_SUM operation 289512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 289612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum( 289712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, 289812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt, const unsigned char *scf, size_t len, 289912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin enum sum_check_flags *pqres, unsigned long flags) 290012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 290112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 290212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *iter; 290312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t pdest, qdest; 290412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op, idst, dst_cnt; 290512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 290612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 290712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 290812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_PQ_DISABLE_P) 290912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pdest = 0; 291012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 291112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pdest = pq[0]; 291212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 291312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_PQ_DISABLE_Q) 291412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qdest = 0; 291512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 291612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qdest = pq[1]; 291712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 291812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id, 291912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src, src_cnt, scf)); 292012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 292112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Always use WXOR for P/Q calculations (two destinations). 292212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Need 1 or 2 extra slots to verify results are zero. 292312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 292412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idst = dst_cnt = (pdest && qdest) ? 2 : 1; 292512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 292612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* One additional slot per destination to clone P/Q 292712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * before calculation (we have to preserve destinations). 292812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 292912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = src_cnt + dst_cnt * 2; 293012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = 1; 293112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 293212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 293312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 293412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op); 293512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 293612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt); 293712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 293812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup byte count for each slot just allocated */ 293912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 294012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 294112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 294212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len); 294312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 294412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 294512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 294612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (pdest) { 294712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 294812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 294912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 295012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = sw_desc->group_head; 295112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 295212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 295312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 295412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 295512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 295612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 295712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 295812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->src_cnt = 0; 295912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->dst_cnt = 0; 296012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 0, 296112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->pdest, 0); 296212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest); 296312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 296412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len); 296512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = 0; 296612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* override pdest to preserve original P */ 296712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pdest = ppc440spe_chan->pdest; 296812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 296912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (qdest) { 297012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 297112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 297212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 297312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&sw_desc->group_list, 297412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 297512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 297612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 297712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 297812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (pdest) { 297912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 298012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 298112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 298212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 298312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 298412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 298512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 298612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 298712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 298812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 298912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 299012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->src_cnt = 0; 299112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->dst_cnt = 0; 299212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 0, 299312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->qdest, 0); 299412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest); 299512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 299612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len); 299712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = 0; 299812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* override qdest to preserve original Q */ 299912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qdest = ppc440spe_chan->qdest; 300012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 300112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 300212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup destinations for P/Q ops */ 300312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest); 300412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 300512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup zero QWORDs into DCHECK CDBs */ 300612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idst = dst_cnt; 300712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_reverse(iter, &sw_desc->group_list, 300812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 300912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 301012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * The last CDB corresponds to Q-parity check, 301112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the one before last CDB corresponds 301212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * P-parity check 301312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 301412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (idst == DMA_DEST_MAX_NUM) { 301512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (idst == dst_cnt) { 301612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_QCHECK, 301712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags); 301812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 301912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_PCHECK, 302012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags); 302112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 302212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 302312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (qdest) { 302412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_QCHECK, 302512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags); 302612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 302712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_PCHECK, 302812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags); 302912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 303012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 303112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->xor_check_result = pqres; 303212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 303312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 303412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * set it to zero, if check fail then result will 303512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * be updated 303612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 303712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *iter->xor_check_result = 0; 303812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dcheck(iter, ppc440spe_chan, 303912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_qword); 304012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 304112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(--dst_cnt)) 304212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 304312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 304412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 304512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup sources and mults for P/Q ops */ 304612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_continue_reverse(iter, &sw_desc->group_list, 304712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 304812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 304912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 mult_dst; 305012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 305112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 305212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 305312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB, 305412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src[src_cnt - 1]); 305512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (qdest) { 305612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 : 305712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1; 305812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, 305912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_MULT1_OFF, 306012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst, 306112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin scf[src_cnt - 1]); 306212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 306312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(--src_cnt)) 306412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 306512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 306612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 306712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 306812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 306912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 307012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 307112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 307212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for 307312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * XOR ZERO_SUM operation 307412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 307512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum( 307612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt, 307712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size_t len, enum sum_check_flags *result, unsigned long flags) 307812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 307912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_async_tx_descriptor *tx; 308012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t pq[2]; 308112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 308212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* validate P, disable Q */ 308312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pq[0] = src[0]; 308412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pq[1] = 0; 308512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags |= DMA_PREP_PQ_DISABLE_Q; 308612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 308712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1], 308812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt - 1, 0, len, 308912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin result, flags); 309012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return tx; 309112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 309212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 309312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 309412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_set_dest - set destination address into descriptor 309512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 309612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc, 309712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index) 309812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 309912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 310012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 310112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(index >= sw_desc->dst_cnt); 310212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 310312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 310412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 310512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 310612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 310712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 310812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* to do: support transfers lengths > 310912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT 311012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 311112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(sw_desc->group_head, 311212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan, 0, addr, index); 311312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 311412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 311512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_get_group_entry(sw_desc, index); 311612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(sw_desc, 311712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan, 0, addr, index); 311812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 311912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 312012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 312112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 312212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter, 312312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, dma_addr_t addr) 312412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 312512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* To clear destinations update the descriptor 312612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (P or Q depending on index) as follows: 312712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * addr is destination (0 corresponds to SG2): 312812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 312912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0); 313012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 313112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* ... and the addr is source: */ 313212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr); 313312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 313412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* addr is always SG2 then the mult is always DST1 */ 313512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 313612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1, 1); 313712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 313812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 313912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 314012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_pq_set_dest - set destination address into descriptor 314112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for the PQXOR operation 314212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 314312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc, 314412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *addrs, unsigned long flags) 314512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 314612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 314712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 314812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t paddr, qaddr; 314912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr = 0, ppath, qpath; 315012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index = 0, i; 315112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 315212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 315312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 315412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_PQ_DISABLE_P) 315512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr = 0; 315612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 315712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr = addrs[0]; 315812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 315912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_PQ_DISABLE_Q) 316012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qaddr = 0; 316112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 316212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qaddr = addrs[1]; 316312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 316412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!paddr || !qaddr) 316512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr = paddr ? paddr : qaddr; 316612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 316712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 316812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 316912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 317012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* walk through the WXOR source list and set P/Q-destinations 317112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for each slot: 317212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 317312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 317412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is WXOR-only chain; may have 1/2 zero descs */ 317512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 317612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index++; 317712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 317812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index++; 317912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 318012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, index); 318112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (addr) { 318212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* one destination */ 318312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, 318412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list, chain_node) 318512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 318612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, addr, 0); 318712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 318812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* two destinations */ 318912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, 319012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list, chain_node) { 319112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 319212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, paddr, 0); 319312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 319412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, qaddr, 1); 319512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 319612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 319712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 319812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index) { 319912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* To clear destinations update the descriptor 320012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (1st,2nd, or both depending on flags) 320112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 320212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index = 0; 320312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, 320412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) { 320512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry( 320612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc, index++); 320712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_zero_op(iter, chan, 320812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr); 320912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 321012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 321112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, 321212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) { 321312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry( 321412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc, index++); 321512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_zero_op(iter, chan, 321612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qaddr); 321712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 321812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 321912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return; 322012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 322112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 322212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is RXOR-only or RXOR/WXOR mixed chain */ 322312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 322412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* If we want to include destination into calculations, 322512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * then make dest addresses cued with mult=1 (XOR). 322612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 322712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ? 322812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB : 322912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE | 323012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << DMA_CUED_MULT1_OFF); 323112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ? 323212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB : 323312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE | 323412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << DMA_CUED_MULT1_OFF); 323512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 323612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup destination(s) in RXOR slot(s) */ 323712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, index++); 323812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 323912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr ? ppath : qpath, 324012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr ? paddr : qaddr, 0); 324112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addr) { 324212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* two destinations */ 324312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 324412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index++); 324512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 324612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qpath, qaddr, 0); 324712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 324812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 324912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) { 325012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup destination(s) in remaining WXOR 325112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * slots 325212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 325312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 325412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index); 325512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (addr) { 325612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* one destination */ 325712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, 325812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list, 325912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) 326012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr( 326112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter, chan, 326212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, 326312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr, 0); 326412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 326512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 326612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* two destinations */ 326712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, 326812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list, 326912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 327012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr( 327112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter, chan, 327212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, 327312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr, 0); 327412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr( 327512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter, chan, 327612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, 327712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qaddr, 1); 327812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 327912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 328012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 328112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 328212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 328312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 328412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 328512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 328612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA2 descriptors have only 1 destination, so there are 328712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * two chains - one for each dest. 328812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * If we want to include destination into calculations, 328912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * then make dest addresses cued with mult=1 (XOR). 329012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 329112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ? 329212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB : 329312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE | 329412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << DMA_CUED_MULT1_OFF); 329512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 329612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ? 329712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB : 329812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE | 329912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << DMA_CUED_MULT1_OFF); 330012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 330112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 0); 330212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < sw_desc->descs_per_op; i++) { 330312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 330412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr ? ppath : qpath, 330512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr ? paddr : qaddr, 0); 330612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 330712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 330812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 330912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 331012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 331112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addr) { 331212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Two destinations; setup Q here */ 331312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 331412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->descs_per_op); 331512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < sw_desc->descs_per_op; i++) { 331612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, 331712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan, qpath, qaddr, 0); 331812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 331912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 332012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 332112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 332212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 332312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 332412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 332512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 332612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 332712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 332812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 332912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor 333012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for the PQ_ZERO_SUM operation 333112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 333212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pqzero_sum_set_dest( 333312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, 333412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t paddr, dma_addr_t qaddr) 333512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 333612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter, *end; 333712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 333812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr = 0; 333912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int idx; 334012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 334112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 334212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 334312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* walk through the WXOR source list and set P/Q-destinations 334412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for each slot 334512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 334612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idx = (paddr && qaddr) ? 2 : 1; 334712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set end */ 334812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_reverse(end, &sw_desc->group_list, 334912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 335012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(--idx)) 335112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 335212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 335312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set start */ 335412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idx = (paddr && qaddr) ? 2 : 1; 335512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, idx); 335612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 335712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (paddr && qaddr) { 335812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* two destinations */ 335912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &sw_desc->group_list, 336012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 336112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(iter == end)) 336212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 336312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 336412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, paddr, 0); 336512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 336612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, qaddr, 1); 336712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 336812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 336912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* one destination */ 337012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr = paddr ? paddr : qaddr; 337112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &sw_desc->group_list, 337212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 337312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(iter == end)) 337412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 337512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 337612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, addr, 0); 337712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 337812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 337912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 338012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* The remaining descriptors are DATACHECK. These have no need in 338112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * destination. Actually, these destinations are used there 338212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * as sources for check operation. So, set addr as source. 338312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 338412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr); 338512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 338612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addr) { 338712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin end = list_entry(end->chain_node.next, 338812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 338912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr); 339012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 339112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 339212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 339312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 339412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor 339512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 339612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline void ppc440spe_desc_set_xor_src_cnt( 339712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 339812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt) 339912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 340012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *hw_desc = desc->hw_desc; 340112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 340212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc &= ~XOR_CDCR_OAC_MSK; 340312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc |= src_cnt; 340412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 340512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 340612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 340712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_pq_set_src - set source address into descriptor 340812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 340912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc, 341012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index) 341112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 341212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 341312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t haddr = 0; 341412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter = NULL; 341512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 341612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 341712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 341812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 341912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 342012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 342112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain 342212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 342312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 342412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR-only or RXOR/WXOR operation */ 342512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int iskip = test_bit(PPC440SPE_DESC_RXOR12, 342612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags) ? 2 : 3; 342712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 342812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == 0) { 342912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 1st slot (RXOR) */ 343012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* setup sources region (R1-2-3, R1-2-4, 343112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * or R1-2-5) 343212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 343312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_RXOR12, 343412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) 343512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_RXOR12 << 343612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 343712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else if (test_bit(PPC440SPE_DESC_RXOR123, 343812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) 343912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_RXOR123 << 344012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 344112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else if (test_bit(PPC440SPE_DESC_RXOR124, 344212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) 344312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_RXOR124 << 344412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 344512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else if (test_bit(PPC440SPE_DESC_RXOR125, 344612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) 344712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_RXOR125 << 344812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 344912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 345012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 345112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr |= DMA_CUED_XOR_BASE; 345212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 0); 345312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (index < iskip) { 345412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 1st slot (RXOR) 345512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * shall actually set source address only once 345612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * instead of first <iskip> 345712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 345812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = NULL; 345912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 346012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 2nd/3d and next slots (WXOR); 346112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * skip first slot with RXOR 346212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 346312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_CUED_XOR_HB; 346412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 346512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index - iskip + sw_desc->dst_cnt); 346612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 346712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 346812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int znum = 0; 346912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 347012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR-only operation; skip first slots with 347112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * zeroing destinations 347212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 347312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 347412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin znum++; 347512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 347612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin znum++; 347712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 347812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_CUED_XOR_HB; 347912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 348012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index + znum); 348112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 348212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 348312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(iter)) { 348412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr); 348512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 348612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!index && 348712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) && 348812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt == 2) { 348912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* if we have two destinations for RXOR, then 349012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * setup source in the second descr too 349112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 349212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 1); 349312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 349412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr, addr); 349512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 349612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 349712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 349812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 349912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 350012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA2 may do Biskup */ 350112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = sw_desc->group_head; 350212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->dst_cnt == 2) { 350312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* both P & Q calculations required; set P src here */ 350412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_set_src(iter, index, addr); 350512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 350612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is for Q */ 350712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 350812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->descs_per_op); 350912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 351012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_set_src(iter, index, addr); 351112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 351212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 351312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 351412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 351512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 351612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor 351712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 351812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_memcpy_xor_set_src( 351912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, 352012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index) 352112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 352212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 352312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 352412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 352512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = sw_desc->group_head; 352612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 352712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(sw_desc)) 352812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr); 352912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 353012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 353112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 353212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_inc_addr - 353312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 353412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_inc_addr( 353512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 353612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_rxor *cursor, int index, int src_cnt) 353712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 353812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count++; 353912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt - 1) { 354012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count); 354112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cursor->addr_count == XOR_MAX_OPS) { 354212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count); 354312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count = 0; 354412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->desc_count++; 354512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 354612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 354712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 354812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 354912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB 355012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 355112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_dma2rxor_prep_src( 355212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *hdesc, 355312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_rxor *cursor, int index, 355412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, u32 addr) 355512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 355612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int rval = 0; 355712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 sign; 355812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc = hdesc; 355912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 356012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 356112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < cursor->desc_count; i++) { 356212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc = list_entry(hdesc->chain_node.next, 356312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 356412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 356512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 356612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 356712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (cursor->state) { 356812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 0: 356912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (addr == cursor->addrl + cursor->len) { 357012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct RXOR */ 357112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 1; 357212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count++; 357312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 357412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 357512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 357612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR12 << DMA_CUED_REGION_OFF); 357712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 357812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 357912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 358012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cursor->addrl == addr + cursor->len) { 358112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse RXOR */ 358212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 1; 358312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count++; 358412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(cursor->addr_count, &desc->reverse_flags[0]); 358512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 358612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 358712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 358812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR12 << DMA_CUED_REGION_OFF); 358912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 359012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 359112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 359212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 359312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "Cannot build " 359412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "DMA2 RXOR command block.\n"); 359512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 359612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 359712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 359812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 1: 359912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sign = test_bit(cursor->addr_count, 360012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->reverse_flags) 360112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ? -1 : 1; 360212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-2 || (sign == -1 360312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin && addr != cursor->addrl - 2*cursor->len)) { 360412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 0; 360512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 1; 360612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addrl = addr; 360712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 360812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 360912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR12 << DMA_CUED_REGION_OFF); 361012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 361112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 361212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (addr == cursor->addrl + 2*sign*cursor->len) { 361312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 2; 361412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 0; 361512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 361612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 361712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR123 << DMA_CUED_REGION_OFF); 361812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 361912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 362012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 362112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 362212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (addr == cursor->addrl + 3*cursor->len) { 362312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 2; 362412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 0; 362512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 362612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 362712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR124 << DMA_CUED_REGION_OFF); 362812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 362912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 363012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 363112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 363212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (addr == cursor->addrl + 4*cursor->len) { 363312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 2; 363412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 0; 363512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 363612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 363712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR125 << DMA_CUED_REGION_OFF); 363812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 363912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 364012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 364112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 364212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 364312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 0; 364412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 1; 364512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addrl = addr; 364612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 364712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 364812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR12 << DMA_CUED_REGION_OFF); 364912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 365012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 365112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 365212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 365312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 2: 365412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 0; 365512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addrl = addr; 365612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count++; 365712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index) { 365812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 365912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 366012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 366112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 366212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 366312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 366412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return rval; 366512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 366612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 366712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 366812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that 366912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call 367012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 367112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_set_src( 367212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 367312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index, dma_addr_t addr) 367412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 367512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 367612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int k = 0, op = 0, lop = 0; 367712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 367812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* get the RXOR operand which corresponds to index addr */ 367912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (op <= index) { 368012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin lop = op; 368112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (k == XOR_MAX_OPS) { 368212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin k = 0; 368312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc = list_entry(desc->chain_node.next, 368412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 368512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb = desc->hw_desc; 368612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 368712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 368812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) == 368912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (DMA_RXOR12 << DMA_CUED_REGION_OFF)) 369012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op += 2; 369112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 369212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op += 3; 369312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 369412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 369512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(k < 1); 369612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 369712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(k-1, desc->reverse_flags)) { 369812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse operand order; put last op in RXOR group */ 369912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == op - 1) 370012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_src(desc, k - 1, addr); 370112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 370212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct operand order; put first op in RXOR group */ 370312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == lop) 370412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_src(desc, k - 1, addr); 370512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 370612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 370712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 370812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 370912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that 371012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call 371112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 371212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_set_mult( 371312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 371412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index, u8 mult) 371512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 371612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 371712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int k = 0, op = 0, lop = 0; 371812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 371912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* get the RXOR operand which corresponds to index mult */ 372012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (op <= index) { 372112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin lop = op; 372212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (k == XOR_MAX_OPS) { 372312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin k = 0; 372412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc = list_entry(desc->chain_node.next, 372512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 372612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 372712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb = desc->hw_desc; 372812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 372912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 373012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) == 373112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (DMA_RXOR12 << DMA_CUED_REGION_OFF)) 373212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op += 2; 373312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 373412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op += 3; 373512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 373612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 373712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(k < 1); 373812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(k-1, desc->reverse_flags)) { 373912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse order */ 374012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult); 374112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 374212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct order */ 374312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult); 374412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 374512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 374612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 374712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 374812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_init_rxor_cursor - 374912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 375012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor) 375112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 375212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(cursor, 0, sizeof(struct ppc440spe_rxor)); 375312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 2; 375412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 375512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 375612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 375712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into 375812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor for the PQXOR operation 375912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 376012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pq_set_src_mult( 376112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, 376212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned char mult, int index, int dst_pos) 376312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 376412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 376512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 mult_idx, mult_dst; 376612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL; 376712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 376812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 376912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 377012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 377112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 377212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 377312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 377412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int region = test_bit(PPC440SPE_DESC_RXOR12, 377512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags) ? 2 : 3; 377612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 377712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index < region) { 377812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR multipliers */ 377912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 378012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt - 1); 378112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc->dst_cnt == 2) 378212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter1 = ppc440spe_get_group_entry( 378312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc, 0); 378412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 378512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx = DMA_CUED_MULT1_OFF + (index << 3); 378612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst = DMA_CDB_SG_SRC; 378712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 378812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR multiplier */ 378912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 379012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index - region + 379112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt); 379212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx = DMA_CUED_MULT1_OFF; 379312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst = dst_pos ? DMA_CDB_SG_DST2 : 379412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1; 379512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 379612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 379712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int znum = 0; 379812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 379912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR-only; 380012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * skip first slots with destinations (if ZERO_DST has 380112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * place) 380212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 380312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 380412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin znum++; 380512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 380612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin znum++; 380712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 380812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, index + znum); 380912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx = DMA_CUED_MULT1_OFF; 381012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1; 381112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 381212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 381312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(iter)) { 381412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, 381512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx, mult_dst, mult); 381612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 381712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(iter1)) { 381812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* if we have two destinations for RXOR, then 381912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * we've just set Q mult. Set-up P now. 382012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 382112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter1, chan, 382212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx, mult_dst, 1); 382312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 382412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 382512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 382612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 382712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 382812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 382912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = sw_desc->group_head; 383012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc->dst_cnt == 2) { 383112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* both P & Q calculations required; set P mult here */ 383212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_set_mult(iter, index, 1); 383312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 383412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* and then set Q mult */ 383512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 383612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->descs_per_op); 383712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 383812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_set_mult(iter, index, mult); 383912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 384012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 384112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 384212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 384312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 384412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_free_chan_resources - free the resources allocated 384512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 384612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_free_chan_resources(struct dma_chan *chan) 384712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 384812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 384912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter, *_iter; 385012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int in_use_descs = 0; 385112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 385212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 385312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_slot_cleanup(ppc440spe_chan); 385412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 385512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 385612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain, 385712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 385812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin in_use_descs++; 385912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&iter->chain_node); 386012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 386112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe_reverse(iter, _iter, 386212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &ppc440spe_chan->all_slots, slot_node) { 386312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&iter->slot_node); 386412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(iter); 386512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->slots_allocated--; 386612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 386712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->last_used = NULL; 386812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 386912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 387012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d %s slots_allocated %d\n", 387112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, 387212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, ppc440spe_chan->slots_allocated); 387312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 387412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 387512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* one is ok since we left it on there on purpose */ 387612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (in_use_descs > 1) 387712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n", 387812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin in_use_descs - 1); 387912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 388012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 388112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 38820793448187643b50af89d36b08470baf45a3cab4Linus Walleij * ppc440spe_adma_tx_status - poll the status of an ADMA transaction 388312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @chan: ADMA channel handle 388412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @cookie: ADMA transaction identifier 38850793448187643b50af89d36b08470baf45a3cab4Linus Walleij * @txstate: a holder for the current state of the channel 388612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 38870793448187643b50af89d36b08470baf45a3cab4Linus Walleijstatic enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan, 38880793448187643b50af89d36b08470baf45a3cab4Linus Walleij dma_cookie_t cookie, struct dma_tx_state *txstate) 388912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 389012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 389112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin enum dma_status ret; 389212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 389312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 389496a2af41c78b1fbb1f567a3486bdc63f7b31c5fdRussell King - ARM Linux ret = dma_cookie_status(chan, cookie, txstate); 389512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret == DMA_SUCCESS) 389612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 389712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 389812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_slot_cleanup(ppc440spe_chan); 389912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 390096a2af41c78b1fbb1f567a3486bdc63f7b31c5fdRussell King - ARM Linux return dma_cookie_status(chan, cookie, txstate); 390112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 390212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 390312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 390412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_eot_handler - end of transfer interrupt handler 390512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 390612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data) 390712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 390812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan = data; 390912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 391012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, 391112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s\n", chan->device->id, __func__); 391212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 391312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_schedule(&chan->irq_tasklet); 391412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_device_clear_eot_status(chan); 391512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 391612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return IRQ_HANDLED; 391712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 391812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 391912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 392012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_err_handler - DMA error interrupt handler; 392112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * do the same things as a eot handler 392212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 392312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic irqreturn_t ppc440spe_adma_err_handler(int irq, void *data) 392412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 392512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan = data; 392612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 392712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, 392812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s\n", chan->device->id, __func__); 392912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 393012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_schedule(&chan->irq_tasklet); 393112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_device_clear_eot_status(chan); 393212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 393312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return IRQ_HANDLED; 393412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 393512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 393612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 393712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_test_callback - called when test operation has been done 393812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 393912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_test_callback(void *unused) 394012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 394112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin complete(&ppc440spe_r6_test_comp); 394212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 394312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 394412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 394512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w 394612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 394712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_issue_pending(struct dma_chan *chan) 394812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 394912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 395012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 395112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 395212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 395312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id, 395412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, ppc440spe_chan->pending); 395512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 395612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_chan->pending) { 395712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->pending = 0; 395812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_append(ppc440spe_chan); 395912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 396012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 396112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 396212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 396312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines 396412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * use FIFOs (as opposite to chains used in XOR) so this is a XOR 396512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * specific operation) 396612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 396712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan) 396812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 396912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 397012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t cookie; 397112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op; 397212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 397312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, 397412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s\n", chan->device->id, __func__); 397512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 397612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&chan->lock); 397712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op); 397812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op); 397912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 398012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 398112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_splice_init(&sw_desc->group_list, &chan->chain); 398212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin async_tx_ack(&sw_desc->async_tx); 398312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_null_xor(group_start); 398412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 39852a926e46022ad7a03e0ac167d8c2b0d88c12c5a8Russell King - ARM Linux cookie = dma_cookie_assign(&sw_desc->async_tx); 398612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 398712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* initialize the completed cookie to be less than 398812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the most recently used cookie 398912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 39904d4e58de32a192fea65ab84509d17d199bd291c8Russell King - ARM Linux chan->common.completed_cookie = cookie - 1; 399112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 399212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* channel should not be busy */ 399312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(ppc440spe_chan_is_busy(chan)); 399412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 399512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set the descriptor address */ 399612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc); 399712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 399812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* run the descriptor */ 399912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_run(chan); 400012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 400112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "ppc440spe adma%d" 400212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " failed to allocate null descriptor\n", 400312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id); 400412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 400512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 400612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 400712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 400812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully. 400912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * For this we just perform one WXOR operation with the same source 401012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * and destination addresses, the GF-multiplier is 1; so if RAID-6 401112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * capabilities are enabled then we'll get src/dst filled with zero. 401212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 401312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan) 401412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 401512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *iter; 401612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct page *pg; 401712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin char *a; 401812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t dma_addr, addrs[2]; 401912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 402012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int rval = 0; 402112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 402212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_WXOR, &op); 402312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 402412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pg = alloc_page(GFP_KERNEL); 402512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!pg) 402612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENOMEM; 402712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 402812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&chan->lock); 402912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1); 403012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 403112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 1 src, 1 dsr, int_ena, WXOR */ 403212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op); 403312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 403412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE); 403512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = PAGE_SIZE; 403612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 403712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 403812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rval = -EFAULT; 403912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 404012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto exit; 404112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 404212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 404312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 404412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Fill the test page with ones */ 404512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(page_address(pg), 0xFF, PAGE_SIZE); 404612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr = dma_map_page(chan->device->dev, pg, 0, 404712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 404812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 404912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup addresses */ 405012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0); 405112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0); 405212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addrs[0] = dma_addr; 405312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addrs[1] = 0; 405412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q); 405512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 405612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin async_tx_ack(&sw_desc->async_tx); 405712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.callback = ppc440spe_test_callback; 405812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.callback_param = NULL; 405912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 406012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin init_completion(&ppc440spe_r6_test_comp); 406112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 406212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_tx_submit(&sw_desc->async_tx); 406312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_issue_pending(&chan->common); 406412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 406512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin wait_for_completion(&ppc440spe_r6_test_comp); 406612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 406712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Now check if the test page is zeroed */ 406812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin a = page_address(pg); 406912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) { 407012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* page is zero - RAID-6 enabled */ 407112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rval = 0; 407212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 407312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RAID-6 was not enabled */ 407412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rval = -EINVAL; 407512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 407612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinexit: 407712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(pg); 407812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return rval; 407912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 408012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 408112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev) 408212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 408312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (adev->id) { 408412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 408512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 408612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_MEMCPY, adev->common.cap_mask); 408712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask); 408812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_PQ, adev->common.cap_mask); 408912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask); 409012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask); 409112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 409212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 409312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_XOR, adev->common.cap_mask); 409412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_PQ, adev->common.cap_mask); 409512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask); 409612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.cap_mask = adev->common.cap_mask; 409712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 409812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 409912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 410012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set base routines */ 410112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_alloc_chan_resources = 410212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_alloc_chan_resources; 410312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_free_chan_resources = 410412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_free_chan_resources; 41050793448187643b50af89d36b08470baf45a3cab4Linus Walleij adev->common.device_tx_status = ppc440spe_adma_tx_status; 410612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_issue_pending = ppc440spe_adma_issue_pending; 410712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 410812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set prep routines based on capability */ 410912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) { 411012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_memcpy = 411112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_memcpy; 411212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 411312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) { 411412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_xor = XOR_MAX_OPS; 411512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_xor = 411612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_xor; 411712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 411812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) { 411912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (adev->id) { 412012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 412112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_set_maxpq(&adev->common, 412212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0); 412312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 412412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 412512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_set_maxpq(&adev->common, 412612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0); 412712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 412812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 412912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_pq = XOR_MAX_OPS * 3; 413012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 413112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 413212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_pq = 413312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_pq; 413412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 413512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) { 413612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (adev->id) { 413712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 413812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_pq = DMA0_FIFO_SIZE / 413912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sizeof(struct dma_cdb); 414012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 414112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 414212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_pq = DMA1_FIFO_SIZE / 414312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sizeof(struct dma_cdb); 414412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 414512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 414612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_pq_val = 414712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_pqzero_sum; 414812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 414912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) { 415012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (adev->id) { 415112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 415212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_xor = DMA0_FIFO_SIZE / 415312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sizeof(struct dma_cdb); 415412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 415512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 415612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_xor = DMA1_FIFO_SIZE / 415712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sizeof(struct dma_cdb); 415812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 415912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 416012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_xor_val = 416112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_xor_zero_sum; 416212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 416312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) { 416412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_interrupt = 416512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_interrupt; 416612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 416712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: " 416812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "( %s%s%s%s%s%s%s)\n", 416912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_name(adev->dev), 417012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "", 417112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "", 417212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "", 417312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "", 417412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "", 417512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : ""); 417612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 417712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 417812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev, 417912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 418012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int *initcode) 418112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 41822dc11581376829303b98eadb2de253bee065a56aGrant Likely struct platform_device *ofdev; 418312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct device_node *np; 418412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int ret; 418512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 41862dc11581376829303b98eadb2de253bee065a56aGrant Likely ofdev = container_of(adev->dev, struct platform_device, dev); 41873e6b02d9f5a9715f7d4ff9e0978e5f9cef53d31fDan Williams np = ofdev->dev.of_node; 418812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id != PPC440SPE_XOR_ID) { 418912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->err_irq = irq_of_parse_and_map(np, 1); 419012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->err_irq == NO_IRQ) { 419112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_warn(adev->dev, "no err irq resource?\n"); 419212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *initcode = PPC_ADMA_INIT_IRQ2; 419312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->err_irq = -ENXIO; 419412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 419512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin atomic_inc(&ppc440spe_adma_err_irq_ref); 419612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 419712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->err_irq = -ENXIO; 419812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 419912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 420012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->irq = irq_of_parse_and_map(np, 0); 420112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->irq == NO_IRQ) { 420212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(adev->dev, "no irq resource\n"); 420312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *initcode = PPC_ADMA_INIT_IRQ1; 420412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENXIO; 420512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_irq_map; 420612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 420712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(adev->dev, "irq %d, err irq %d\n", 420812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->irq, adev->err_irq); 420912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 421012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = request_irq(adev->irq, ppc440spe_adma_eot_handler, 421112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 0, dev_driver_string(adev->dev), chan); 421212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) { 421312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(adev->dev, "can't request irq %d\n", 421412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->irq); 421512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *initcode = PPC_ADMA_INIT_IRQ1; 421612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -EIO; 421712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_req1; 421812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 421912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 422012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* only DMA engines have a separate error IRQ 422112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * so it's Ok if err_irq < 0 in XOR engine case. 422212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 422312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->err_irq > 0) { 422412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* both DMA engines share common error IRQ */ 422512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = request_irq(adev->err_irq, 422612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_err_handler, 422712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin IRQF_SHARED, 422812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_driver_string(adev->dev), 422912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan); 423012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) { 423112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(adev->dev, "can't request irq %d\n", 423212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->err_irq); 423312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *initcode = PPC_ADMA_INIT_IRQ2; 423412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -EIO; 423512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_req2; 423612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 423712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 423812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 423912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) { 424012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* enable XOR engine interrupts */ 424112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT | 424212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT, 424312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &adev->xor_reg->ier); 424412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 424512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 mask, enable; 424612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 424712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe"); 424812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!np) { 424912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't find I2O device tree node\n", 425012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 425112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 425212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_req2; 425312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 425412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->i2o_reg = of_iomap(np, 0); 425512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!adev->i2o_reg) { 425612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to map I2O registers\n", __func__); 425712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 425812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -EINVAL; 425912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_req2; 426012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 426112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 426212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Unmask 'CS FIFO Attention' interrupts and 426312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * enable generating interrupts on errors 426412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 426512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin enable = (adev->id == PPC440SPE_DMA0_ID) ? 426612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) : 426712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM); 426812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mask = ioread32(&adev->i2o_reg->iopim) & enable; 426912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(mask, &adev->i2o_reg->iopim); 427012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 427112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 427212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 427312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_req2: 427412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin free_irq(adev->irq, chan); 427512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_req1: 427612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin irq_dispose_mapping(adev->irq); 427712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_irq_map: 427812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->err_irq > 0) { 427912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) 428012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin irq_dispose_mapping(adev->err_irq); 428112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 428212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 428312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 428412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 428512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev, 428612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 428712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 428812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 mask, disable; 428912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 429012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) { 429112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* disable XOR engine interrupts */ 429212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mask = ioread32be(&adev->xor_reg->ier); 429312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT | 429412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT); 429512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(mask, &adev->xor_reg->ier); 429612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 429712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* disable DMAx engine interrupts */ 429812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin disable = (adev->id == PPC440SPE_DMA0_ID) ? 429912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) : 430012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM); 430112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mask = ioread32(&adev->i2o_reg->iopim) | disable; 430212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(mask, &adev->i2o_reg->iopim); 430312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 430412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin free_irq(adev->irq, chan); 430512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin irq_dispose_mapping(adev->irq); 430612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->err_irq > 0) { 430712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin free_irq(adev->err_irq, chan); 430812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) { 430912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin irq_dispose_mapping(adev->err_irq); 431012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->i2o_reg); 431112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 431212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 431312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 431412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 431512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 431612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_probe - probe the asynch device 431712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 4318463a1f8b3ceebe990ca9a8c7cc2e51ee42cb48ebBill Pembertonstatic int ppc440spe_adma_probe(struct platform_device *ofdev) 431912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 432005c02542c20aa00dc9a66f4bfb1a89d1131457f2Anatolij Gustschin struct device_node *np = ofdev->dev.of_node; 432112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct resource res; 432212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_device *adev; 432312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 432412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc_dma_chan_ref *ref, *_ref; 432512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int ret = 0, initcode = PPC_ADMA_INIT_OK; 432612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const u32 *idx; 432712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int len; 432812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin void *regs; 432912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 id, pool_size; 433012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 433112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (of_device_is_compatible(np, "amcc,xor-accelerator")) { 433212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin id = PPC440SPE_XOR_ID; 433312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* As far as the XOR engine is concerned, it does not 433412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * use FIFOs but uses linked list. So there is no dependency 433512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * between pool size to allocate and the engine configuration. 433612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 433712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pool_size = PAGE_SIZE << 1; 433812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 433912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* it is DMA0 or DMA1 */ 434012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idx = of_get_property(np, "cell-index", &len); 434112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!idx || (len != sizeof(u32))) { 434212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "Device node %s has missing " 434312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "or invalid cell-index property\n", 434412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np->full_name); 434512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 434612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 434712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin id = *idx; 434812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA0,1 engines use FIFO to maintain CDBs, so we 434912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * should allocate the pool accordingly to size of this 435012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * FIFO. Thus, the pool size depends on the FIFO depth: 435112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * how much CDBs pointers the FIFO may contain then so 435212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * much CDBs we should provide in the pool. 435312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * That is 435412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * CDB size = 32B; 435512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * CDBs number = (DMA0_FIFO_SIZE >> 3); 435612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Pool size = CDBs number * CDB size = 435712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2. 435812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 435912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pool_size = (id == PPC440SPE_DMA0_ID) ? 436012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA0_FIFO_SIZE : DMA1_FIFO_SIZE; 436112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pool_size <<= 2; 436212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 436312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 436412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (of_address_to_resource(np, 0, &res)) { 436512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to get memory resource\n"); 436612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_MEMRES; 436712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 436812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out; 436912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 437012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 437112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!request_mem_region(res.start, resource_size(&res), 437212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_driver_string(&ofdev->dev))) { 4373a584bff5efae8c1d026e3a930e3d13a90264fafcJoe Perches dev_err(&ofdev->dev, "failed to request memory region %pR\n", 4374a584bff5efae8c1d026e3a930e3d13a90264fafcJoe Perches &res); 437512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_MEMREG; 437612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -EBUSY; 437712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out; 437812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 437912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 438012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* create a device */ 438112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev = kzalloc(sizeof(*adev), GFP_KERNEL); 438212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!adev) { 438312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to allocate device\n"); 438412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_ALLOC; 438512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 438612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_adev_alloc; 438712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 438812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 438912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->id = id; 439012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->pool_size = pool_size; 439112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* allocate coherent memory for hardware descriptors */ 439212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev, 439312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->pool_size, &adev->dma_desc_pool, 439412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin GFP_KERNEL); 439512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->dma_desc_pool_virt == NULL) { 439612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent " 439712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "memory for hardware descriptors\n", 439812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->pool_size); 439912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_COHERENT; 440012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 440112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_dma_alloc; 440212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 4403d73111c6d43f08d697a3d986fe3bd8ca796a2096Masanari Iida dev_dbg(&ofdev->dev, "allocated descriptor pool virt 0x%p phys 0x%llx\n", 440412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool); 440512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 440612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin regs = ioremap(res.start, resource_size(&res)); 440712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!regs) { 440812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to ioremap regs!\n"); 440912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_regs_alloc; 441012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 441112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 441212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) { 441312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->xor_reg = regs; 441412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Reset XOR */ 441512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr); 441612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr); 441712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 441812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ? 441912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA0_FIFO_SIZE : DMA1_FIFO_SIZE; 442012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_reg = regs; 442112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMAx_FIFO_SIZE is defined in bytes, 442212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * <fsiz> - is defined in number of CDB pointers (8byte). 442312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * DMA FIFO Length = CSlength + CPlength, where 442412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * CSlength = CPlength = (fsiz + 1) * 8. 442512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 442612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2), 442712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &adev->dma_reg->fsiz); 442812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Configure DMA engine */ 442912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN, 443012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &adev->dma_reg->cfg); 443112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Clear Status */ 443212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(~0, &adev->dma_reg->dsts); 443312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 443412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 443512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dev = &ofdev->dev; 443612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.dev = &ofdev->dev; 443712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&adev->common.channels); 4438dd3daca162f7411448dd80a8872a002c43cfd8e5Jingoo Han platform_set_drvdata(ofdev, adev); 443912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 444012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* create a channel */ 444112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = kzalloc(sizeof(*chan), GFP_KERNEL); 444212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!chan) { 444312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "can't allocate channel structure\n"); 444412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_CHANNEL; 444512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 444612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_chan_alloc; 444712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 444812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 444912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_init(&chan->lock); 445012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&chan->chain); 445112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&chan->all_slots); 445212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device = adev; 445312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->common.device = &adev->common; 44548ac695463f37af902e953d575d3f782e32e170daRussell King - ARM Linux dma_cookie_init(&chan->common); 445512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_add_tail(&chan->common.device_node, &adev->common.channels); 445612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_init(&chan->irq_tasklet, ppc440spe_adma_tasklet, 445712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (unsigned long)chan); 445812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 445912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* allocate and map helper pages for async validation or 446012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * async_mult/async_sum_product operations on DMA0/1. 446112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 446212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id != PPC440SPE_XOR_ID) { 446312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->pdest_page = alloc_page(GFP_KERNEL); 446412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->qdest_page = alloc_page(GFP_KERNEL); 446512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!chan->pdest_page || 446612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin !chan->qdest_page) { 446712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan->pdest_page) 446812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(chan->pdest_page); 446912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan->qdest_page) 447012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(chan->qdest_page); 447112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 447212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_page_alloc; 447312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 447412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0, 447512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 447612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0, 447712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 447812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 447912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 448012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ref = kmalloc(sizeof(*ref), GFP_KERNEL); 448112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ref) { 448212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ref->chan = &chan->common; 448312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&ref->node); 448412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_add_tail(&ref->node, &ppc440spe_adma_chan_list); 448512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 448612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to allocate channel reference!\n"); 448712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 448812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_ref_alloc; 448912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 449012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 449112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode); 449212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) 449312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_irq; 449412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 449512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_init_capabilities(adev); 449612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 449712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = dma_async_device_register(&adev->common); 449812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) { 449912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_REGISTER; 450012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to register dma device\n"); 450112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_dev_reg; 450212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 450312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 450412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out; 450512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 450612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_dev_reg: 450712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_release_irqs(adev, chan); 450812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_irq: 450912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) { 451012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan == to_ppc440spe_adma_chan(ref->chan)) { 451112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&ref->node); 451212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ref); 451312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 451412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 451512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_ref_alloc: 451612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id != PPC440SPE_XOR_ID) { 451712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(&ofdev->dev, chan->pdest, 451812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 451912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(&ofdev->dev, chan->qdest, 452012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 452112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(chan->pdest_page); 452212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(chan->qdest_page); 452312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 452412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_page_alloc: 452512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(chan); 452612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_chan_alloc: 452712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) 452812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->xor_reg); 452912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 453012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->dma_reg); 453112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_regs_alloc: 453212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_free_coherent(adev->dev, adev->pool_size, 453312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool_virt, 453412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool); 453512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_dma_alloc: 453612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(adev); 453712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_adev_alloc: 453812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin release_mem_region(res.start, resource_size(&res)); 453912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout: 454012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (id < PPC440SPE_ADMA_ENGINES_NUM) 454112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_devices[id] = initcode; 454212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 454312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 454412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 454512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 454612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 454712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_remove - remove the asynch device 454812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 45494bf27b8b333bcd291664fd0f7d129099d474a23bGreg Kroah-Hartmanstatic int ppc440spe_adma_remove(struct platform_device *ofdev) 455012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 4551dd3daca162f7411448dd80a8872a002c43cfd8e5Jingoo Han struct ppc440spe_adma_device *adev = platform_get_drvdata(ofdev); 455205c02542c20aa00dc9a66f4bfb1a89d1131457f2Anatolij Gustschin struct device_node *np = ofdev->dev.of_node; 455312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct resource res; 455412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, *_chan; 455512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc_dma_chan_ref *ref, *_ref; 455612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 455712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 455812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id < PPC440SPE_ADMA_ENGINES_NUM) 455912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_devices[adev->id] = -1; 456012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 456112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_async_device_unregister(&adev->common); 456212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 456312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(chan, _chan, &adev->common.channels, 456412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin device_node) { 456512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 456612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_release_irqs(adev, ppc440spe_chan); 456712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_kill(&ppc440spe_chan->irq_tasklet); 456812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id != PPC440SPE_XOR_ID) { 456912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest, 457012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 457112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest, 457212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 457312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(ppc440spe_chan->pdest_page); 457412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(ppc440spe_chan->qdest_page); 457512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 457612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, 457712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin node) { 457812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_chan == 457912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin to_ppc440spe_adma_chan(ref->chan)) { 458012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&ref->node); 458112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ref); 458212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 458312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 458412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&chan->device_node); 458512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ppc440spe_chan); 458612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 458712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 458812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_free_coherent(adev->dev, adev->pool_size, 458912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool_virt, adev->dma_desc_pool); 459012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) 459112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->xor_reg); 459212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 459312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->dma_reg); 459412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_address_to_resource(np, 0, &res); 459512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin release_mem_region(res.start, resource_size(&res)); 459612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(adev); 459712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 459812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 459912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 460012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* 460112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * /sys driver interface to enable h/w RAID-6 capabilities 460212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/ 460312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * directory are "devices", "enable" and "poly". 460412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * "devices" shows available engines. 460512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * "enable" is used to enable RAID-6 capabilities or to check 460612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * whether these has been activated. 460712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * "poly" allows setting/checking used polynomial (for PPC440SPe only). 460812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 460912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 461012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t show_ppc440spe_devices(struct device_driver *dev, char *buf) 461112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 461212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ssize_t size = 0; 461312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 461412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 461512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) { 461612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_adma_devices[i] == -1) 461712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 461812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size += snprintf(buf + size, PAGE_SIZE - size, 461912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "PPC440SP(E)-ADMA.%d: %s\n", i, 462012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc_adma_errors[ppc440spe_adma_devices[i]]); 462112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 462212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return size; 462312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 462412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 462512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t show_ppc440spe_r6enable(struct device_driver *dev, char *buf) 462612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 462712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return snprintf(buf, PAGE_SIZE, 462812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "PPC440SP(e) RAID-6 capabilities are %sABLED.\n", 462912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_r6_enabled ? "EN" : "DIS"); 463012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 463112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 463212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t store_ppc440spe_r6enable(struct device_driver *dev, 463312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const char *buf, size_t count) 463412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 463512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long val; 463612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 463712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!count || count > 11) 463812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 463912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 464012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!ppc440spe_r6_tchan) 464112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EFAULT; 464212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 464312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Write a key */ 464412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sscanf(buf, "%lx", &val); 464512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val); 464612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin isync(); 464712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 464812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Verify whether it really works now */ 464912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) { 465012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_info("PPC440SP(e) RAID-6 has been activated " 465112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "successfully\n"); 465212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_r6_enabled = 1; 465312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 465412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_info("PPC440SP(e) RAID-6 hasn't been activated!" 465512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " Error key ?\n"); 465612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_r6_enabled = 0; 465712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 465812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return count; 465912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 466012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 466112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t show_ppc440spe_r6poly(struct device_driver *dev, char *buf) 466212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 466312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ssize_t size = 0; 466412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 reg; 466512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 466612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#ifdef CONFIG_440SP 466712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 440SP has fixed polynomial */ 466812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg = 0x4d; 466912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#else 467012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL); 467112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg >>= MQ0_CFBHL_POLY; 467212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg &= 0xFF; 467312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#endif 467412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 467512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver " 467612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "uses 0x1%02x polynomial.\n", reg); 467712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return size; 467812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 467912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 468012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t store_ppc440spe_r6poly(struct device_driver *dev, 468112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const char *buf, size_t count) 468212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 468312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long reg, val; 468412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 468512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#ifdef CONFIG_440SP 468612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 440SP uses default 0x14D polynomial only */ 468712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 468812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#endif 468912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 469012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!count || count > 6) 469112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 469212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 469312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* e.g., 0x14D or 0x11D */ 469412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sscanf(buf, "%lx", &val); 469512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 469612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (val & ~0x1FF) 469712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 469812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 469912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin val &= 0xFF; 470012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL); 470112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg &= ~(0xFF << MQ0_CFBHL_POLY); 470212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg |= val << MQ0_CFBHL_POLY; 470312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg); 470412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 470512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return count; 470612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 470712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 470812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic DRIVER_ATTR(devices, S_IRUGO, show_ppc440spe_devices, NULL); 470912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic DRIVER_ATTR(enable, S_IRUGO | S_IWUSR, show_ppc440spe_r6enable, 471012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin store_ppc440spe_r6enable); 471112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic DRIVER_ATTR(poly, S_IRUGO | S_IWUSR, show_ppc440spe_r6poly, 471212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin store_ppc440spe_r6poly); 471312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 471412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* 471512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Common initialisation for RAID engines; allocate memory for 471612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * DMAx FIFOs, perform configuration common for all DMA engines. 471712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Further DMA engine specific configuration is done at probe time. 471812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 471912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_configure_raid_devices(void) 472012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 472112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct device_node *np; 472212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct resource i2o_res; 472312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct i2o_regs __iomem *i2o_reg; 472412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_host_t i2o_dcr_host; 472512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int dcr_base, dcr_len; 472612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i, ret; 472712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 472812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe"); 472912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!np) { 473012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't find I2O device tree node\n", 473112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 473212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENODEV; 473312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 473412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 473512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (of_address_to_resource(np, 0, &i2o_res)) { 473612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 473712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 473812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 473912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 474012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i2o_reg = of_iomap(np, 0); 474112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!i2o_reg) { 474212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to map I2O registers\n", __func__); 474312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 474412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 474512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 474612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 474712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Get I2O DCRs base */ 474812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_base = dcr_resource_start(np, 0); 474912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_len = dcr_resource_len(np, 0); 475012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!dcr_base && !dcr_len) { 475112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't get DCR registers base/len!\n", 475212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np->full_name); 475312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 475412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(i2o_reg); 475512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENODEV; 475612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 475712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 475812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i2o_dcr_host = dcr_map(np, dcr_base, dcr_len); 475912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!DCR_MAP_OK(i2o_dcr_host)) { 476012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to map DCRs!\n", np->full_name); 476112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 476212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(i2o_reg); 476312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENODEV; 476412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 476512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 476612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 476712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share 476812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the base address of FIFO memory space. 476912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Actually we need twice more physical memory than programmed in the 477012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * <fsiz> register (because there are two FIFOs for each DMA: CP and CS) 477112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 477212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1, 477312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin GFP_KERNEL); 477412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!ppc440spe_dma_fifo_buf) { 477512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__); 477612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(i2o_reg); 477712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_unmap(i2o_dcr_host, dcr_len); 477812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENOMEM; 477912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 478012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 478112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 478212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Configure h/w 478312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 478412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Reset I2O/DMA */ 478512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA); 478612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mtdcri(SDR0, DCRN_SDR0_SRST, 0); 478712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 478812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup the base address of mmaped registers */ 478912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32)); 479012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) | 479112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin I2O_REG_ENABLE); 479212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_unmap(i2o_dcr_host, dcr_len); 479312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 479412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup FIFO memory space base address */ 479512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(0, &i2o_reg->ifbah); 479612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal); 479712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 479812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set zero FIFO size for I2O, so the whole 479912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma_fifo_buf is used by DMAs. 480012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * DMAx_FIFOs will be configured while probe. 480112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 480212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(0, &i2o_reg->ifsiz); 480312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(i2o_reg); 480412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 480512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* To prepare WXOR/RXOR functionality we need access to 480612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Memory Queue Module DCRs (finally it will be enabled 480712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * via /sys interface of the ppc440spe ADMA driver). 480812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 480912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe"); 481012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!np) { 481112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't find MQ device tree node\n", 481212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 481312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 481412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_free; 481512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 481612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 481712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Get MQ DCRs base */ 481812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_base = dcr_resource_start(np, 0); 481912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_len = dcr_resource_len(np, 0); 482012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!dcr_base && !dcr_len) { 482112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't get DCR registers base/len!\n", 482212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np->full_name); 482312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 482412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_mq; 482512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 482612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 482712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len); 482812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) { 482912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to map DCRs!\n", np->full_name); 483012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 483112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_mq; 483212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 483312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 483412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_mq_dcr_len = dcr_len; 483512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 483612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set HB alias */ 483712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB); 483812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 483912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set: 484012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - LL transaction passing limit to 1; 484112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - Memory controller cycle limit to 1; 484212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - Galois Polynomial to 0x14d (default) 484312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 484412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, 484512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) | 484612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY)); 484712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 484812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin atomic_set(&ppc440spe_adma_err_irq_ref, 0); 484912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) 485012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_devices[i] = -1; 485112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 485212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 485312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 485412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_mq: 485512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 485612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_free: 485712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ppc440spe_dma_fifo_buf); 485812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 485912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 486012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 48614bf27b8b333bcd291664fd0f7d129099d474a23bGreg Kroah-Hartmanstatic const struct of_device_id ppc440spe_adma_of_match[] = { 486212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin { .compatible = "ibm,dma-440spe", }, 486312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin { .compatible = "amcc,xor-accelerator", }, 486412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin {}, 486512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 486612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinMODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match); 486712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 4868000061245a6797d542854106463b6b20fbdcb12eGrant Likelystatic struct platform_driver ppc440spe_adma_driver = { 486912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin .probe = ppc440spe_adma_probe, 4870a7d6e3ec28bba30d1409d70de1958edc6d9109d7Bill Pemberton .remove = ppc440spe_adma_remove, 487112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin .driver = { 487212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin .name = "PPC440SP(E)-ADMA", 487312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin .owner = THIS_MODULE, 48744018294b53d1dae026880e45f174c1cc63b5d435Grant Likely .of_match_table = ppc440spe_adma_of_match, 487512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin }, 487612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 487712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 487812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic __init int ppc440spe_adma_init(void) 487912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 488012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int ret; 488112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 488212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = ppc440spe_configure_raid_devices(); 488312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) 488412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 488512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 4886000061245a6797d542854106463b6b20fbdcb12eGrant Likely ret = platform_driver_register(&ppc440spe_adma_driver); 488712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) { 488812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to register platform driver\n", 488912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 489012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_reg; 489112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 489212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 489312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Initialization status */ 489412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = driver_create_file(&ppc440spe_adma_driver.driver, 489512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_devices); 489612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) 489712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_dev; 489812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 489912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RAID-6 h/w enable entry */ 490012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = driver_create_file(&ppc440spe_adma_driver.driver, 490112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_enable); 490212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) 490312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_en; 490412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 490512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* GF polynomial to use */ 490612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = driver_create_file(&ppc440spe_adma_driver.driver, 490712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_poly); 490812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!ret) 490912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 491012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 491112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 491212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_enable); 491312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_en: 491412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 491512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_devices); 491612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_dev: 491712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* User will not be able to enable h/w RAID-6 */ 491812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to create RAID-6 driver interface\n", 491912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 4920000061245a6797d542854106463b6b20fbdcb12eGrant Likely platform_driver_unregister(&ppc440spe_adma_driver); 492112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_reg: 492212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len); 492312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ppc440spe_dma_fifo_buf); 492412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 492512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 492612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 492712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void __exit ppc440spe_adma_exit(void) 492812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 492912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 493012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_poly); 493112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 493212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_enable); 493312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 493412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_devices); 4935000061245a6797d542854106463b6b20fbdcb12eGrant Likely platform_driver_unregister(&ppc440spe_adma_driver); 493612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len); 493712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ppc440spe_dma_fifo_buf); 493812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 493912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 494012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinarch_initcall(ppc440spe_adma_init); 494112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinmodule_exit(ppc440spe_adma_exit); 494212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 494312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinMODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>"); 494412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinMODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver"); 494512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinMODULE_LICENSE("GPL"); 4946