112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* 212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Copyright (C) 2006-2009 DENX Software Engineering. 312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Author: Yuri Tikhonov <yur@emcraft.com> 512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Further porting to arch/powerpc by 712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Anatolij Gustschin <agust@denx.de> 812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * This program is free software; you can redistribute it and/or modify it 1012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * under the terms of the GNU General Public License as published by the Free 1112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Software Foundation; either version 2 of the License, or (at your option) 1212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * any later version. 1312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 1412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * This program is distributed in the hope that it will be useful, but WITHOUT 1512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 1612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 1712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * more details. 1812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 1912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * You should have received a copy of the GNU General Public License along with 2012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * this program; if not, write to the Free Software Foundation, Inc., 59 2112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Temple Place - Suite 330, Boston, MA 02111-1307, USA. 2212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 2312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * The full GNU General Public License is included in this distribution in the 2412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * file called COPYING. 2512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 2612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 2712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* 2812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * This driver supports the asynchrounous DMA copy and RAID engines available 2912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * on the AMCC PPC440SPe Processors. 3012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) 3112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ADMA driver written by D.Williams. 3212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 3312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 3412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/init.h> 3512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/module.h> 3612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/async_tx.h> 3712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/delay.h> 3812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/dma-mapping.h> 3912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/spinlock.h> 4012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/interrupt.h> 415a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 4212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/uaccess.h> 4312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/proc_fs.h> 4412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/of.h> 45c11eede69b6ad0ac44ebc1e021a8d2699c5f1f8fRob Herring#include <linux/of_address.h> 46c11eede69b6ad0ac44ebc1e021a8d2699c5f1f8fRob Herring#include <linux/of_irq.h> 4712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/of_platform.h> 4812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <asm/dcr.h> 4912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <asm/dcr-regs.h> 5012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include "adma.h" 51d2ebfb335b0426deb1a4fb14e4e926d81ecd8235Russell King - ARM Linux#include "../dmaengine.h" 5212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 5312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinenum ppc_adma_init_code { 5412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_OK = 0, 5512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_MEMRES, 5612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_MEMREG, 5712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_ALLOC, 5812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_COHERENT, 5912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_CHANNEL, 6012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_IRQ1, 6112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_IRQ2, 6212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_REGISTER 6312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 6412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 6512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic char *ppc_adma_errors[] = { 6612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_OK] = "ok", 6712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource", 6812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_MEMREG] = "failed to request memory region", 6912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev " 7012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "structure", 7112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for " 7212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "hardware descriptors", 7312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel", 7412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_IRQ1] = "failed to request first irq", 7512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_IRQ2] = "failed to request second irq", 7612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device", 7712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 7812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 7912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic enum ppc_adma_init_code 8012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM]; 8112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 8212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstruct ppc_dma_chan_ref { 8312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan; 8412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct list_head node; 8512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 8612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 8712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* The list of channels exported by ppc440spe ADMA */ 8812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstruct list_head 8912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list); 9012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 9112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* This flag is set when want to refetch the xor chain in the interrupt 9212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * handler 9312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 9412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 do_xor_refetch; 9512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 9612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* Pointer to DMA0, DMA1 CP/CS FIFO */ 9712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void *ppc440spe_dma_fifo_buf; 9812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 9912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* Pointers to last submitted to DMA0, DMA1 CDBs */ 10012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *chan_last_sub[3]; 10112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *chan_first_cdb[3]; 10212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 10312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* Pointer to last linked and submitted xor CB */ 10412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *xor_last_linked; 10512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *xor_last_submit; 10612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 10712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* This array is used in data-check operations for storing a pattern */ 10812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic char ppc440spe_qword[16]; 10912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 11012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic atomic_t ppc440spe_adma_err_irq_ref; 11112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic dcr_host_t ppc440spe_mq_dcr_host; 11212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic unsigned int ppc440spe_mq_dcr_len; 11312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 11412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* Since RXOR operations use the common register (MQ0_CF2H) for setting-up 11512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the block size in transactions, then we do not allow to activate more than 11612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * only one RXOR transactions simultaneously. So use this var to store 11712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is 11812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * set) or not (PPC440SPE_RXOR_RUN is clear). 11912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 12012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic unsigned long ppc440spe_rxor_state; 12112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 12212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* These are used in enable & check routines 12312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 12412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 ppc440spe_r6_enabled; 12512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_chan *ppc440spe_r6_tchan; 12612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct completion ppc440spe_r6_test_comp; 12712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 12812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_dma2rxor_prep_src( 12912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 13012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_rxor *cursor, int index, 13112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, u32 addr); 13212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_set_src( 13312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 13412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index, dma_addr_t addr); 13512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_set_mult( 13612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 13712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index, u8 mult); 13812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 13912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#ifdef ADMA_LL_DEBUG 14012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define ADMA_LL_DBG(x) ({ if (1) x; 0; }) 14112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#else 14212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define ADMA_LL_DBG(x) ({ if (0) x; 0; }) 14312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#endif 14412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 14512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void print_cb(struct ppc440spe_adma_chan *chan, void *block) 14612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 14712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *cdb; 14812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *cb; 14912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 15012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 15112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 15212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 0: 15312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 1: 15412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb = block; 15512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 15612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("CDB at %p [%d]:\n" 15712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n" 15812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t sg1u 0x%08x sg1l 0x%08x\n" 15912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t sg2u 0x%08x sg2l 0x%08x\n" 16012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t sg3u 0x%08x sg3l 0x%08x\n", 16112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb, chan->device->id, 16212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt), 16312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l), 16412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l), 16512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l) 16612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ); 16712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 16812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 2: 16912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb = block; 17012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 17112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("CB at %p [%d]:\n" 17212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n" 17312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t cbtah 0x%08x cbtal 0x%08x\n" 17412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t cblah 0x%08x cblal 0x%08x\n", 17512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb, chan->device->id, 17612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb->cbc, cb->cbbc, cb->cbs, 17712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb->cbtah, cb->cbtal, 17812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb->cblah, cb->cblal); 17912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < 16; i++) { 18012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i && !cb->ops[i].h && !cb->ops[i].l) 18112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 18212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n", 18312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i, cb->ops[i].h, cb->ops[i].l); 18412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 18512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 18612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 18712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 18812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 18912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void print_cb_list(struct ppc440spe_adma_chan *chan, 19012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter) 19112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 19212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (; iter; iter = iter->hw_next) 19312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin print_cb(chan, iter->hw_desc); 19412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 19512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 19612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src, 19712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt) 19812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 19912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 20012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 20112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\n%s(%d):\nsrc: ", __func__, id); 20212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 20312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx ", src[i]); 20412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("dst:\n\t0x%016llx\n", dst); 20512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 20612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 20712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src, 20812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt) 20912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 21012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 21112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 21212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\n%s(%d):\nsrc: ", __func__, id); 21312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 21412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx ", src[i]); 21512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("dst: "); 21612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < 2; i++) 21712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx ", dst[i]); 21812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 21912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 22012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src, 22112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt, 22212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf) 22312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 22412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 22512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 22612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id); 22712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (scf) { 22812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 22912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]); 23012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 23112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 23212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx(no) ", src[i]); 23312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 23412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 23512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("dst: "); 23612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < 2; i++) 23712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx ", src[src_cnt + i]); 23812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 23912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 24012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/****************************************************************************** 24112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Command (Descriptor) Blocks low-level routines 24212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ******************************************************************************/ 24312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 24412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT 24512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * pseudo operation 24612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 24712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc, 24812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 24912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 25012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *p; 25112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 25212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 25312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 25412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin p = desc->hw_desc; 25512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 25612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* NOP with Command Block Complete Enable */ 25712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin p->cbc = XOR_CBCR_CBCE_BIT; 25812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 25912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 26012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 26112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 26212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* NOP with interrupt */ 26312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &desc->flags); 26412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 26512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 26612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id, 26712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 26812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 26912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 27012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 27112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 27212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 27312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR 27412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * pseudo operation 27512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 27612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc) 27712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 27812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 27912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 28012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = 0; 28112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = 1; 28212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 28312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 28412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 28512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation 28612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 28712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc, 28812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, unsigned long flags) 28912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 29012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *hw_desc = desc->hw_desc; 29112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 29212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 29312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 29412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = src_cnt; 29512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = 1; 29612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 29712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt; 29812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 29912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Enable interrupt on completion */ 30012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc |= XOR_CBCR_CBCE_BIT; 30112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 30212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 30312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 30412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ 30512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * operation in DMA2 controller 30612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 30712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc, 30812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int dst_cnt, int src_cnt, unsigned long flags) 30912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 31012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *hw_desc = desc->hw_desc; 31112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 31212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 31312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 31412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = src_cnt; 31512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = dst_cnt; 31612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags)); 31712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->descs_per_op = 0; 31812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 31912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc = XOR_CBCR_TGT_BIT; 32012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 32112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Enable interrupt on completion */ 32212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc |= XOR_CBCR_CBCE_BIT; 32312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 32412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 32512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define DMA_CTRL_FLAGS_LAST DMA_PREP_FENCE 32612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define DMA_PREP_ZERO_P (DMA_CTRL_FLAGS_LAST << 1) 32712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define DMA_PREP_ZERO_Q (DMA_PREP_ZERO_P << 1) 32812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 32912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 33012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation 33112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * with DMA0/1 33212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 33312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc, 33412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int dst_cnt, int src_cnt, unsigned long flags, 33512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op) 33612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 33712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 33812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 33912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 dopc; 34012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 34112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Common initialization of a PQ descriptors chain */ 34212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bits(op, &desc->flags); 34312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = src_cnt; 34412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = dst_cnt; 34512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 34612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR MULTICAST if both P and Q are being computed 34712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * MV_SG1_SG2 if Q only 34812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 34912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ? 35012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2; 35112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 35212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &desc->group_list, chain_node) { 35312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 35412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 35512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 35612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(!list_is_last(&iter->chain_node, 35712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &desc->group_list))) { 35812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 35912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 36012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 36112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 36212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 36312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is the last descriptor. 36412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * this slot will be pasted from ADMA level 36512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * each time it wants to configure parameters 36612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * of the transaction (src, dst, ...) 36712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 36812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 36912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 37012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 37112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 37212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 37312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 37412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 37512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 37612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set OPS depending on WXOR/RXOR type of operation */ 37712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) { 37812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is a WXOR only chain: 37912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - first descriptors are for zeroing destinations 38012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * if PPC440SPE_ZERO_P/Q set; 38112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - descriptors remained are for GF-XOR operations. 38212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 38312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&desc->group_list, 38412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 38512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 38612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 38712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) { 38812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 38912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 39012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 39112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 39212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 39312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 39412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 39512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) { 39612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 39712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 39812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 39912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 40012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 40112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 40212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 40312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &desc->group_list, chain_node) { 40412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 40512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = dopc; 40612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 40712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 40812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is either RXOR-only or mixed RXOR/WXOR */ 40912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 41012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* The first 1 or 2 slots in chain are always RXOR, 41112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * if need to calculate P & Q, then there are two 41212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * RXOR slots; if only P or only Q, then there is one 41312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 41412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&desc->group_list, 41512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 41612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 41712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 41812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 41912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 42012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (desc->dst_cnt == DMA_DEST_MAX_NUM) { 42112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 42212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 42312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 42412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 42512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 42612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 42712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 42812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* The remaining descs (if any) are WXORs */ 42912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) { 43012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 43112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 43212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 43312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &desc->group_list, 43412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 43512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 43612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = dopc; 43712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 43812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 43912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 44012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 44112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 44212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 44312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor 44412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for PQ_ZERO_SUM operation 44512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 44612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_dma01pqzero_sum( 44712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 44812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int dst_cnt, int src_cnt) 44912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 45012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 45112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 45212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i = 0; 45312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST : 45412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_OPC_MV_SG1_SG2; 45512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 45612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Initialize starting from 2nd or 3rd descriptor dependent 45712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * on dst_cnt. First one or two slots are for cloning P 45812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * and/or Q to chan->pdest and/or chan->qdest as we have 45912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * to preserve original P/Q. 46012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 46112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&desc->group_list, 46212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 46312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 46412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 46512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 46612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dst_cnt > 1) { 46712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 46812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 46912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 47012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* initialize each source descriptor in chain */ 47112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &desc->group_list, chain_node) { 47212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 47312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 47412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->src_cnt = 0; 47512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->dst_cnt = 0; 47612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 47712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is a ZERO_SUM operation: 47812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - <src_cnt> descriptors starting from 2nd or 3rd 47912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor are for GF-XOR operations; 48012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - remaining <dst_cnt> descriptors are for checking the result 48112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 48212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i++ < src_cnt) 48312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* MV_SG1_SG2 if only Q is being verified 48412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * MULTICAST if both P and Q are being verified 48512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 48612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = dopc; 48712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 48812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA_CDB_OPC_DCHECK128 operation */ 48912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_DCHECK128; 49012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 49112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(!list_is_last(&iter->chain_node, 49212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &desc->group_list))) { 49312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 49412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 49512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 49612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 49712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 49812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is the last descriptor. 49912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * this slot will be pasted from ADMA level 50012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * each time it wants to configure parameters 50112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * of the transaction (src, dst, ...) 50212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 50312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 50412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* always enable interrupt generation since we get 50512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the status of pqzero from the handler 50612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 50712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 50812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 50912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 51012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = src_cnt; 51112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = dst_cnt; 51212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 51312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 51412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 51512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation 51612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 51712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc, 51812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long flags) 51912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 52012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc = desc->hw_desc; 52112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 52212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 52312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 52412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = 1; 52512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = 1; 52612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 52712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 52812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &desc->flags); 52912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 53012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &desc->flags); 53112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 53212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 53312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 53412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 53512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 53612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_src_addr - set source address into the descriptor 53712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 53812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc, 53912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 54012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_idx, dma_addr_t addrh, 54112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addrl) 54212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 54312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 54412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 54512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin phys_addr_t addr64, tmplow, tmphi; 54612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 54712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 54812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 54912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 55012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addrh) { 55112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr64 = addrl; 55212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmphi = (addr64 >> 32); 55312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmplow = (addr64 & 0xFFFFFFFF); 55412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 55512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmphi = addrh; 55612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmplow = addrl; 55712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 55812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 55912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow); 56012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi); 56112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 56212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 56312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 56412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->ops[src_idx].l = addrl; 56512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->ops[src_idx].h |= addrh; 56612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 56712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 56812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 56912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 57012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 57112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_src_mult - set source address mult into the descriptor 57212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 57312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc, 57412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, u32 mult_index, 57512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int sg_index, unsigned char mult_value) 57612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 57712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 57812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 57912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 *psgu; 58012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 58112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 58212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 58312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 58412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 58512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 58612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (sg_index) { 58712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* for RXOR operations set multiplier 58812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * into source cued address 58912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 59012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_SG_SRC: 59112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgu = &dma_hw_desc->sg1u; 59212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 59312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* for WXOR operations set multiplier 59412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * into destination cued address(es) 59512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 59612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_SG_DST1: 59712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgu = &dma_hw_desc->sg2u; 59812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 59912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_SG_DST2: 60012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgu = &dma_hw_desc->sg3u; 60112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 60212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 60312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 60412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 60512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 60612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *psgu |= cpu_to_le32(mult_value << mult_index); 60712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 60812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 60912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 61012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 61112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 61212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 61312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 61412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 61512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 61612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 61712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_dest_addr - set destination address into the descriptor 61812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 61912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc, 62012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 62112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addrh, dma_addr_t addrl, 62212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 dst_idx) 62312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 62412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 62512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 62612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin phys_addr_t addr64, tmphi, tmplow; 62712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 *psgu, *psgl; 62812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 62912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 63012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 63112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 63212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addrh) { 63312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr64 = addrl; 63412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmphi = (addr64 >> 32); 63512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmplow = (addr64 & 0xFFFFFFFF); 63612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 63712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmphi = addrh; 63812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmplow = addrl; 63912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 64012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 64112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 64212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u; 64312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l; 64412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 64512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *psgl = cpu_to_le32((u32)tmplow); 64612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *psgu |= cpu_to_le32((u32)tmphi); 64712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 64812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 64912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 65012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbtal = addrl; 65112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbtah |= addrh; 65212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 65312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 65412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 65512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 65612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 65712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_byte_count - set number of data bytes involved 65812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * into the operation 65912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 66012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc, 66112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 66212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 byte_count) 66312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 66412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 66512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 66612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 66712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 66812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 66912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 67012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 67112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->cnt = cpu_to_le32(byte_count); 67212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 67312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 67412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 67512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbbc = byte_count; 67612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 67712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 67812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 67912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 68012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 68112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_rxor_block_size - set RXOR block size 68212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 68312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count) 68412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 68512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* assume that byte_count is aligned on the 512-boundary; 68612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * thus write it directly to the register (bits 23:31 are 68712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * reserved there). 68812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 68912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count); 69012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 69112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 69212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 69312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_dcheck - set CHECK pattern 69412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 69512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc, 69612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, u8 *qword) 69712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 69812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 69912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 70012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 70112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 70212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 70312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 70412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(qword[0], &dma_hw_desc->sg3l); 70512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(qword[4], &dma_hw_desc->sg3u); 70612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(qword[8], &dma_hw_desc->sg2l); 70712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(qword[12], &dma_hw_desc->sg2u); 70812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 70912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 71012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 71112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 71212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 71312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 71412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 71512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_xor_set_link - set link address in xor CB 71612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 71712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc, 71812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *next_desc) 71912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 72012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc = prev_desc->hw_desc; 72112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 72212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!next_desc || !(next_desc->phys))) { 72312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n", 72412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, next_desc, 72512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin next_desc ? next_desc->phys : 0); 72612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 72712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 72812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 72912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbs = 0; 73012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cblal = next_desc->phys; 73112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cblah = 0; 73212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT; 73312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 73412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 73512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 73612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_link - set the address of descriptor following this 73712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor in chain 73812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 73912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan, 74012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *prev_desc, 74112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *next_desc) 74212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 74312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long flags; 74412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *tail = next_desc; 74512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 74612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!prev_desc || !next_desc || 74712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (prev_desc->hw_next && prev_desc->hw_next != next_desc))) { 74812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* If previous next is overwritten something is wrong. 74912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * though we may refetch from append to initiate list 75012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * processing; in this case - it's ok. 75112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 75212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; " 75312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "prev->hw_next=0x%p\n", __func__, prev_desc, 75412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin next_desc, prev_desc ? prev_desc->hw_next : 0); 75512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 75612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 75712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 75812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin local_irq_save(flags); 75912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 76012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do s/w chaining both for DMA and XOR descriptors */ 76112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prev_desc->hw_next = next_desc; 76212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 76312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 76412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 76512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 76612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 76712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 76812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* bind descriptor to the chain */ 76912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (tail->hw_next) 77012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tail = tail->hw_next; 77112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_linked = tail; 77212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 77312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (prev_desc == xor_last_submit) 77412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do not link to the last submitted CB */ 77512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 77612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_xor_set_link(prev_desc, next_desc); 77712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 77812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 77912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 78012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin local_irq_restore(flags); 78112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 78212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 78312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 78412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_get_link - get the address of the descriptor that 78512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * follows this one 78612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 78712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc, 78812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 78912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 79012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!desc->hw_next) 79112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 79212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 79312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return desc->hw_next->phys; 79412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 79512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 79612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 79712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_is_aligned - check alignment 79812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 79912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline int ppc440spe_desc_is_aligned( 80012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, int num_slots) 80112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 80212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return (desc->idx & (num_slots - 1)) ? 0 : 1; 80312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 80412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 80512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 80612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_xor_slot_count - get the number of slots necessary for 80712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * XOR operation 80812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 80912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt, 81012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int *slots_per_op) 81112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 81212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 81312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 81412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* each XOR descriptor provides up to 16 source operands */ 81512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS; 81612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 81712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT)) 81812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return slot_cnt; 81912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 82012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: len %d > max %d !!\n", 82112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 82212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 82312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return slot_cnt; 82412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 82512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 82612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 82712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for 82812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * DMA2 PQ operation 82912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 83012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs, 83112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, size_t len) 83212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 83312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin signed long long order = 0; 83412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int state = 0; 83512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int addr_count = 0; 83612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 83712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 1; i < src_cnt; i++) { 83812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t cur_addr = srcs[i]; 83912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t old_addr = srcs[i-1]; 84012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (state) { 84112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 0: 84212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cur_addr == old_addr + len) { 84312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct RXOR */ 84412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 1; 84512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 1; 84612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 84712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 84812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (old_addr == cur_addr + len) { 84912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse RXOR */ 85012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = -1; 85112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 1; 85212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 85312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 85412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 85512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 3; 85612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 85712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 85812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 1: 85912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-2 || (order == -1 86012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin && cur_addr != old_addr - len)) { 86112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 86212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 86312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 86412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cur_addr == old_addr + len*order) { 86512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 2; 86612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 86712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 86812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cur_addr == old_addr + 2*len) { 86912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 2; 87012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 87112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 87212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cur_addr == old_addr + 3*len) { 87312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 2; 87412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 87512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 87612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 87712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 87812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 87912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 88012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 88112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 88212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 2: 88312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 88412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 88512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 88612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 88712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 88812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (state == 3) 88912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 89012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 89112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt <= 1 || (state != 1 && state != 2)) { 89212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n", 89312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, src_cnt, state, addr_count, order); 89412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 89512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("\t[%d] 0x%llx \n", i, srcs[i]); 89612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 89712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 89812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 89912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS; 90012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 90112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 90212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 90312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/****************************************************************************** 90412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ADMA channel low-level routines 90512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ******************************************************************************/ 90612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 90712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 90812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan); 90912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan); 91012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 91112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 91212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine 91312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 91412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_device_clear_eot_status( 91512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 91612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 91712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_regs *dma_reg; 91812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 91912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 *p = chan->device->dma_desc_pool_virt; 92012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *cdb; 92112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 rv, i; 92212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 92312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 92412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 92512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 92612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* read FIFO to ack */ 92712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_reg = chan->device->dma_reg; 92812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while ((rv = ioread32(&dma_reg->csfpl))) { 92912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i = rv & DMA_CDB_ADDR_MSK; 93012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb = (struct dma_cdb *)&p[i - 93112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (u32)chan->device->dma_desc_pool]; 93212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 93312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Clear opcode to ack. This is necessary for 93412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ZeroSum operations only 93512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 93612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb->opc = 0; 93712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 93812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_RXOR_RUN, 93912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &ppc440spe_rxor_state)) { 94012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* probably this is a completed RXOR op, 94112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * get pointer to CDB using the fact that 94212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * physical and virtual addresses of CDB 94312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * in pools have the same offsets 94412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 94512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (le32_to_cpu(cdb->sg1u) & 94612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE) { 94712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is a RXOR */ 94812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_RXOR_RUN, 94912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &ppc440spe_rxor_state); 95012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 95112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 95212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 95312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rv & DMA_CDB_STATUS_MSK) { 95412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* ZeroSum check failed 95512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 95612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 95712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t phys = rv & ~DMA_CDB_MSK; 95812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 95912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 96012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Update the status of corresponding 96112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor. 96212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 96312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &chan->chain, 96412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 96512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->phys == phys) 96612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 96712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 96812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 96912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * if cannot find the corresponding 97012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * slot it's a bug 97112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 97212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(&iter->chain_node == &chan->chain); 97312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 97412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->xor_check_result) { 97512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_PCHECK, 97612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags)) { 97712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *iter->xor_check_result |= 97812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin SUM_CHECK_P_RESULT; 97912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 98012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_QCHECK, 98112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags)) { 98212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *iter->xor_check_result |= 98312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin SUM_CHECK_Q_RESULT; 98412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 98512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 98612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 98712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 98812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 98912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 99012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rv = ioread32(&dma_reg->dsts); 99112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rv) { 99212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("DMA%d err status: 0x%x\n", 99312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id, rv); 99412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* write back to clear */ 99512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(rv, &dma_reg->dsts); 99612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 99712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 99812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 99912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reset status bits to ack */ 100012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 100112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rv = ioread32be(&xor_reg->sr); 100212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(rv, &xor_reg->sr); 100312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 100412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) { 100512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rv & XOR_IE_RPTIE_BIT) { 100612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Read PLB Timeout Error. 100712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Try to resubmit the CB 100812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 100912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 val = ioread32be(&xor_reg->ccbalr); 101012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 101112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(val, &xor_reg->cblalr); 101212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 101312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin val = ioread32be(&xor_reg->crsr); 101412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(val | XOR_CRSR_XAE_BIT, 101512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &xor_reg->crsr); 101612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 101712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("XOR ERR 0x%x status\n", rv); 101812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 101912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 102012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 102112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* if the XORcore is idle, but there are unprocessed CBs 102212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * then refetch the s/w chain here 102312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 102412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) && 102512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin do_xor_refetch) 102612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_append(chan); 102712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 102812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 102912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 103012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 103112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 103212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_is_busy - get the channel status 103312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 103412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan) 103512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 103612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_regs *dma_reg; 103712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 103812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int busy = 0; 103912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 104012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 104112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 104212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 104312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_reg = chan->device->dma_reg; 104412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* if command FIFO's head and tail pointers are equal and 104512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * status tail is the same as command, then channel is free 104612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 104712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) || 104812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp)) 104912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin busy = 1; 105012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 105112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 105212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* use the special status bit for the XORcore 105312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 105412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 105512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0; 105612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 105712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 105812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 105912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return busy; 106012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 106112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 106212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 106312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_set_first_xor_descriptor - init XORcore chain 106412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 106512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_set_first_xor_descriptor( 106612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 106712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *next_desc) 106812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 106912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg = chan->device->xor_reg; 107012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 107112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) 107212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_INFO "%s: Warn: XORcore is running " 107312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "when try to set the first CDB!\n", 107412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 107512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 107612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_submit = xor_last_linked = next_desc; 107712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 107812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr); 107912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 108012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(next_desc->phys, &xor_reg->cblalr); 108112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(0, &xor_reg->cblahr); 108212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT, 108312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &xor_reg->cbcr); 108412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 108512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->hw_chain_inited = 1; 108612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 108712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 108812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 108912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO. 109012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * called with irqs disabled 109112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 109212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan, 109312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc) 109412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 109512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 pcdb; 109612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_regs *dma_reg = chan->device->dma_reg; 109712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 109812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pcdb = desc->phys; 109912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_INT, &desc->flags)) 110012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pcdb |= DMA_CDB_NO_INT; 110112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 110212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan_last_sub[chan->device->id] = desc; 110312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 110412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(print_cb(chan, desc->hw_desc)); 110512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 110612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(pcdb, &dma_reg->cpfpl); 110712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 110812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 110912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 111012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_append - update the h/w chain in the channel 111112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 111212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan) 111312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 111412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 111512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 111612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb; 111712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 cur_desc; 111812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long flags; 111912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 112012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin local_irq_save(flags); 112112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 112212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 112312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 112412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 112512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cur_desc = ppc440spe_chan_get_current_descriptor(chan); 112612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 112712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(cur_desc)) { 112812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = chan_last_sub[chan->device->id]; 112912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!iter); 113012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 113112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* first peer */ 113212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = chan_first_cdb[chan->device->id]; 113312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!iter); 113412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_dma_put_desc(chan, iter); 113512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->hw_chain_inited = 1; 113612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 113712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 113812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* is there something new to append */ 113912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!iter->hw_next) 114012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 114112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 114212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* flush descriptors from the s/w queue to fifo */ 114312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_continue(iter, &chan->chain, chain_node) { 114412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_dma_put_desc(chan, iter); 114512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!iter->hw_next) 114612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 114712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 114812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 114912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 115012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* update h/w links and refetch */ 115112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!xor_last_submit->hw_next) 115212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 115312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 115412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 115512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* the last linked CDB has to generate an interrupt 115612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * that we'd be able to append the next lists to h/w 115712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * regardless of the XOR engine state at the moment of 115812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * appending of these next lists 115912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 116012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb = xor_last_linked->hw_desc; 116112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->cbc |= XOR_CBCR_CBCE_BIT; 116212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 116312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) { 116412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* XORcore is idle. Refetch now */ 116512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin do_xor_refetch = 0; 116612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_xor_set_link(xor_last_submit, 116712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_submit->hw_next); 116812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 116912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(print_cb_list(chan, 117012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_submit->hw_next)); 117112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 117212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_submit = xor_last_linked; 117312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(ioread32be(&xor_reg->crsr) | 117412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT, 117512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &xor_reg->crsr); 117612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 117712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* XORcore is running. Refetch later in the handler */ 117812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin do_xor_refetch = 1; 117912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 118012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 118112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 118212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 118312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 118412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin local_irq_restore(flags); 118512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 118612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 118712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 118812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor 118912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 119012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 119112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan) 119212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 119312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_regs *dma_reg; 119412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 119512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 119612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!chan->hw_chain_inited)) 119712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* h/w descriptor chain is not initialized yet */ 119812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 119912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 120012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 120112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 120212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 120312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_reg = chan->device->dma_reg; 120412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK); 120512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 120612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 120712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ioread32be(&xor_reg->ccbalr); 120812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 120912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 121012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 121112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 121212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 121312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_run - enable the channel 121412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 121512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan) 121612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 121712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 121812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 121912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 122012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 122112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 122212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMAs are always enabled, do nothing */ 122312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 122412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 122512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* drain write buffer */ 122612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 122712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 122812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* fetch descriptor pointed to in <link> */ 122912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT, 123012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &xor_reg->crsr); 123112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 123212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 123312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 123412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 123512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/****************************************************************************** 123612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ADMA device level 123712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ******************************************************************************/ 123812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 123912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan); 124012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan); 124112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 124212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic dma_cookie_t 124312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx); 124412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 124512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx, 124612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index); 124712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 124812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx, 124912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index); 125012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 125112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 125212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx, 125312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *paddr, unsigned long flags); 125412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 125512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx, 125612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index); 125712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 125812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx, 125912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned char mult, int index, int dst_pos); 126012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 126112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx, 126212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t paddr, dma_addr_t qaddr); 126312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 126412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct page *ppc440spe_rxor_srcs[32]; 126512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 126612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 126712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_can_rxor - check if the operands may be processed with RXOR 126812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 126912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len) 127012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 127112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i, order = 0, state = 0; 127212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int idx = 0; 127312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 127412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!(src_cnt > 1))) 127512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 127612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 127712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs)); 127812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 127912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Skip holes in the source list before checking */ 128012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) { 128112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!srcs[i]) 128212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 128312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_srcs[idx++] = srcs[i]; 128412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 128512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt = idx; 128612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 128712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 1; i < src_cnt; i++) { 128812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin char *cur_addr = page_address(ppc440spe_rxor_srcs[i]); 128912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]); 129012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 129112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (state) { 129212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 0: 129312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cur_addr == old_addr + len) { 129412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct RXOR */ 129512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 1; 129612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 1; 129712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (old_addr == cur_addr + len) { 129812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse RXOR */ 129912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = -1; 130012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 1; 130112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 130212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out; 130312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 130412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 1: 130512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((i == src_cnt - 2) || 130612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (order == -1 && cur_addr != old_addr - len)) { 130712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 130812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 130912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if ((cur_addr == old_addr + len * order) || 131012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (cur_addr == old_addr + 2 * len) || 131112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (cur_addr == old_addr + 3 * len)) { 131212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 2; 131312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 131412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 131512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 131612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 131712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 131812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 2: 131912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 132012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 132112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 132212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 132312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 132412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 132512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout: 132612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (state == 1 || state == 2) 132712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 132812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 132912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 133012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 133112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 133212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 133312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_device_estimate - estimate the efficiency of processing 133412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the operation given on this channel. It's assumed that 'chan' is 133512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * capable to process 'cap' type of operation. 133612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @chan: channel to use 133712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @cap: type of transaction 133812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @dst_lst: array of destination pointers 133912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @dst_cnt: number of destination operands 134012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @src_lst: array of source pointers 134112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @src_cnt: number of source operands 134212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @src_sz: size of each source operand 134312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 134412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_estimate(struct dma_chan *chan, 134512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt, 134612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct page **src_lst, int src_cnt, size_t src_sz) 134712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 134812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int ef = 1; 134912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 135012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cap == DMA_PQ || cap == DMA_PQ_VAL) { 135112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* If RAID-6 capabilities were not activated don't try 135212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * to use them 135312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 135412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!ppc440spe_r6_enabled)) 135512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -1; 135612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 135712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* In the current implementation of ppc440spe ADMA driver it 135812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * makes sense to pick out only pq case, because it may be 135912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * processed: 136012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (1) either using Biskup method on DMA2; 136112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (2) or on DMA0/1. 136212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Thus we give a favour to (1) if the sources are suitable; 136312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * else let it be processed on one of the DMA0/1 engines. 136412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * In the sum_product case where destination is also the 136512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * source process it on DMA0/1 only. 136612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 136712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) { 136812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 136912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1]) 137012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ef = 0; /* sum_product case, process on DMA0/1 */ 137112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz)) 137212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ef = 3; /* override (DMA0/1 + idle) */ 137312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 137412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ef = 0; /* can't process on DMA2 if !rxor */ 137512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 137612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 137712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* channel idleness increases the priority */ 137812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(ef) && 137912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan))) 138012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ef++; 138112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 138212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ef; 138312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 138412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 138512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstruct dma_chan * 138612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap, 138712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct page **dst_lst, int dst_cnt, struct page **src_lst, 138812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, size_t src_sz) 138912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 139012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *best_chan = NULL; 139112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc_dma_chan_ref *ref; 139212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int best_rank = -1; 139312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 139412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!src_sz)) 139512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 139612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_sz > PAGE_SIZE) { 139712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 139812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * should a user of the api ever pass > PAGE_SIZE requests 139912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * we sort out cases where temporary page-sized buffers 140012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * are used. 140112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 140212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (cap) { 140312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_PQ: 140412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt == 1 && dst_lst[1] == src_lst[0]) 140512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 140612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt == 2 && dst_lst[1] == src_lst[1]) 140712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 140812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 140912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_PQ_VAL: 141012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_XOR_VAL: 141112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 141212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 141312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 141412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 141512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 141612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 141712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) { 141812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(cap, ref->chan->device->cap_mask)) { 141912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int rank; 142012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 142112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst, 142212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst_cnt, src_lst, src_cnt, src_sz); 142312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rank > best_rank) { 142412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin best_rank = rank; 142512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin best_chan = ref->chan; 142612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 142712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 142812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 142912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 143012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return best_chan; 143112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 143212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinEXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel); 143312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 143412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 143512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_get_group_entry - get group entry with index idx 143612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @tdesc: is the last allocated slot in the group. 143712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 143812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot * 143912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx) 144012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 144112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter = tdesc->group_head; 144212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i = 0; 144312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 144412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) { 144512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n", 144612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt); 144712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 144812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 144912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 145012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &tdesc->group_list, chain_node) { 145112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i++ == entry_idx) 145212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 145312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 145412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return iter; 145512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 145612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 145712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 145812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_free_slots - flags descriptor slots for reuse 145912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @slot: Slot to free 146012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Caller must hold &ppc440spe_chan->lock while calling this function 146112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 146212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot, 146312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 146412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 146512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int stride = slot->slots_per_op; 146612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 146712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (stride--) { 146812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->slots_per_op = 0; 146912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot = list_entry(slot->slot_node.next, 147012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 147112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node); 147212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 147312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 147412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 147512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 147612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_run_tx_complete_actions - call functions to be called 147712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * upon completion 147812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 147912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic dma_cookie_t ppc440spe_adma_run_tx_complete_actions( 148012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 148112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 148212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t cookie) 148312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 148412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(desc->async_tx.cookie < 0); 148512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (desc->async_tx.cookie > 0) { 148612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = desc->async_tx.cookie; 148712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->async_tx.cookie = 0; 148812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 148912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* call the callback (must not sleep or submit new 149012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * operations to this channel) 149112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 149212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (desc->async_tx.callback) 149312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->async_tx.callback( 149412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->async_tx.callback_param); 149512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 1496d38a8c622a1b382336c3e152c6caf4e11d1f1b2aDan Williams dma_descriptor_unmap(&desc->async_tx); 149712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 149812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 149912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* run dependent operations */ 150012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_run_dependencies(&desc->async_tx); 150112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 150212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return cookie; 150312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 150412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 150512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 150612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set) 150712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 150812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc, 150912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 151012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 151112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* the client is allowed to attach dependent operations 151212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * until 'ack' is set 151312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 151412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!async_tx_test_ack(&desc->async_tx)) 151512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 151612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 151712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* leave the last descriptor in the chain 151812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * so we can append to it 151912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 152012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (list_is_last(&desc->chain_node, &chan->chain) || 152112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->phys == ppc440spe_chan_get_current_descriptor(chan)) 152212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 152312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 152412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan->device->id != PPC440SPE_XOR_ID) { 152512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* our DMA interrupt handler clears opc field of 152612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * each processed descriptor. For all types of 152712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * operations except for ZeroSum we do not actually 152812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * need ack from the interrupt handler. ZeroSum is a 152912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * special case since the result of this operation 153012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * is available from the handler only, so if we see 153112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * such type of descriptor (which is unprocessed yet) 153212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * then leave it in chain. 153312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 153412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *cdb = desc->hw_desc; 153512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cdb->opc == DMA_CDB_OPC_DCHECK128) 153612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 153712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 153812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 153912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n", 154012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->phys, desc->idx, desc->slots_per_op); 154112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 154212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&desc->chain_node); 154312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_free_slots(desc, chan); 154412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 154512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 154612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 154712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 154812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine 154912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * which runs through the channel CDBs list until reach the descriptor 155012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * currently processed. When routine determines that all CDBs of group 155112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * are completed then corresponding callbacks (if any) are called and slots 155212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * are freed. 155312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 155412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan) 155512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 155612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL; 155712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t cookie = 0; 155812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 current_desc = ppc440spe_chan_get_current_descriptor(chan); 155912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int busy = ppc440spe_chan_is_busy(chan); 156012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int seen_current = 0, slot_cnt = 0, slots_per_op = 0; 156112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 156212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n", 156312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id, __func__); 156412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 156512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!current_desc) { 156612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* There were no transactions yet, so 156712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * nothing to clean 156812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 156912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return; 157012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 157112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 157212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* free completed slots from the chain starting with 157312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the oldest descriptor 157412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 157512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(iter, _iter, &chan->chain, 157612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 157712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d " 157812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "busy: %d this_desc: %#llx next_desc: %#x " 157912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "cur: %#x ack: %d\n", 158012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->async_tx.cookie, iter->idx, busy, iter->phys, 158112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_get_link(iter, chan), current_desc, 158212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin async_tx_test_ack(&iter->async_tx)); 158312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prefetch(_iter); 158412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prefetch(&_iter->async_tx); 158512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 158612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do not advance past the current descriptor loaded into the 158712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * hardware channel,subsequent descriptors are either in process 158812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * or have not been submitted 158912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 159012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (seen_current) 159112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 159212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 159312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* stop the search if we reach the current descriptor and the 159412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * channel is busy, or if it appears that the current descriptor 159512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * needs to be re-read (i.e. has been appended to) 159612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 159712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->phys == current_desc) { 159812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(seen_current++); 159912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (busy || ppc440spe_desc_get_link(iter, chan)) { 160012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* not all descriptors of the group have 160112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * been completed; exit. 160212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 160312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 160412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 160512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 160612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 160712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* detect the start of a group transaction */ 160812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!slot_cnt && !slots_per_op) { 160912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = iter->slot_cnt; 161012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = iter->slots_per_op; 161112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slot_cnt <= slots_per_op) { 161212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = 0; 161312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = 0; 161412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 161512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 161612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 161712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slot_cnt) { 161812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!group_start) 161912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = iter; 162012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt -= slots_per_op; 162112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 162212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 162312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* all the members of a group are complete */ 162412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slots_per_op != 0 && slot_cnt == 0) { 162512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter; 162612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int end_of_chain = 0; 162712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 162812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* clean up the group */ 162912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = group_start->slot_cnt; 163012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin grp_iter = group_start; 163112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe_from(grp_iter, _grp_iter, 163212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &chan->chain, chain_node) { 163312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 163412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = ppc440spe_adma_run_tx_complete_actions( 163512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin grp_iter, chan, cookie); 163612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 163712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt -= slots_per_op; 163812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin end_of_chain = ppc440spe_adma_clean_slot( 163912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin grp_iter, chan); 164012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (end_of_chain && slot_cnt) { 164112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Should wait for ZeroSum completion */ 164212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cookie > 0) 16434d4e58de32a192fea65ab84509d17d199bd291c8Russell King - ARM Linux chan->common.completed_cookie = cookie; 164412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return; 164512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 164612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 164712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slot_cnt == 0 || end_of_chain) 164812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 164912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 165012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 165112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* the group should be complete at this point */ 165212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(slot_cnt); 165312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 165412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = 0; 165512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = NULL; 165612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (end_of_chain) 165712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 165812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 165912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 166012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (slots_per_op) /* wait for group completion */ 166112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 166212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 166312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan, 166412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie); 166512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 166612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_adma_clean_slot(iter, chan)) 166712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 166812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 166912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 167012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!seen_current); 167112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 167212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cookie > 0) { 16734d4e58de32a192fea65ab84509d17d199bd291c8Russell King - ARM Linux chan->common.completed_cookie = cookie; 167412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\tcompleted cookie %d\n", cookie); 167512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 167612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 167712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 167812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 167912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 168012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_tasklet - clean up watch-dog initiator 168112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 168212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_tasklet(unsigned long data) 168312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 168412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan = (struct ppc440spe_adma_chan *) data; 168512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 168612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING); 168712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __ppc440spe_adma_slot_cleanup(chan); 168812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock(&chan->lock); 168912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 169012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 169112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 169212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_slot_cleanup - clean up scheduled initiator 169312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 169412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan) 169512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 169612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&chan->lock); 169712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __ppc440spe_adma_slot_cleanup(chan); 169812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 169912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 170012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 170112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 170212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_alloc_slots - allocate free slots (if any) 170312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 170412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots( 170512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, int num_slots, 170612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slots_per_op) 170712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 170812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter = NULL, *_iter; 170912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *alloc_start = NULL; 171012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct list_head chain = LIST_HEAD_INIT(chain); 171112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slots_found, retry = 0; 171212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 171312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 171412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!num_slots || !slots_per_op); 171512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* start search from the last allocated descrtiptor 171612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * if a contiguous allocation can not be found start searching 171712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * from the beginning of the list 171812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 171912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinretry: 172012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_found = 0; 172112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (retry == 0) 172212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = chan->last_used; 172312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 172412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(&chan->all_slots, 172512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 172612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node); 172712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots, 172812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node) { 172912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prefetch(_iter); 173012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prefetch(&_iter->async_tx); 173112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->slots_per_op) { 173212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_found = 0; 173312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 173412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 173512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 173612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* start the allocation if the slot is correctly aligned */ 173712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!slots_found++) 173812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin alloc_start = iter; 173912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 174012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slots_found == num_slots) { 174112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *alloc_tail = NULL; 174212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *last_used = NULL; 174312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 174412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = alloc_start; 174512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (num_slots) { 174612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 174712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* pre-ack all but the last descriptor */ 174812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (num_slots != slots_per_op) 174912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin async_tx_ack(&iter->async_tx); 175012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 175112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_add_tail(&iter->chain_node, &chain); 175212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin alloc_tail = iter; 175312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->async_tx.cookie = 0; 175412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 175512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->flags = 0; 175612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->slot_cnt = num_slots; 175712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->xor_check_result = NULL; 175812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < slots_per_op; i++) { 175912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->slots_per_op = slots_per_op - i; 176012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin last_used = iter; 176112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->slot_node.next, 176212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 176312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node); 176412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 176512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin num_slots -= slots_per_op; 176612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 176712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin alloc_tail->group_head = alloc_start; 176812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin alloc_tail->async_tx.cookie = -EBUSY; 176912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_splice(&chain, &alloc_tail->group_list); 177012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->last_used = last_used; 177112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return alloc_tail; 177212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 177312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 177412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!retry++) 177512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto retry; 177612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 177712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* try to free some slots if the allocation fails */ 177812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_schedule(&chan->irq_tasklet); 177912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 178012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 178112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 178212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 178312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_alloc_chan_resources - allocate pools for CDB slots 178412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 178512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan) 178612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 178712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 178812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *slot = NULL; 178912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin char *hw_desc; 179012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i, db_sz; 179112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int init; 179212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 179312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 179412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin init = ppc440spe_chan->slots_allocated ? 0 : 1; 179512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->chan_id = ppc440spe_chan->device->id; 179612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 179712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Allocate descriptor slots */ 179812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i = ppc440spe_chan->slots_allocated; 179912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID) 180012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin db_sz = sizeof(struct dma_cdb); 180112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 180212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin db_sz = sizeof(struct xor_cb); 180312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 180412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) { 180512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot), 180612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin GFP_KERNEL); 180712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!slot) { 180812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_INFO "SPE ADMA Channel only initialized" 180912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " %d descriptor slots", i--); 181012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 181112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 181212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 181312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt; 181412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->hw_desc = (void *) &hw_desc[i * db_sz]; 181512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_async_tx_descriptor_init(&slot->async_tx, chan); 181612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->async_tx.tx_submit = ppc440spe_adma_tx_submit; 181712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&slot->chain_node); 181812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&slot->slot_node); 181912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&slot->group_list); 182012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz; 182112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->idx = i; 182212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 182312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 182412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->slots_allocated++; 182512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots); 182612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 182712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 182812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 182912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i && !ppc440spe_chan->last_used) { 183012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->last_used = 183112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_entry(ppc440spe_chan->all_slots.next, 183212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 183312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node); 183412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 183512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 183612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 183712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: allocated %d descriptor slots\n", 183812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, i); 183912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 184012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* initialize the channel and the chain with a null operation */ 184112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (init) { 184212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (ppc440spe_chan->device->id) { 184312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 184412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 184512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->hw_chain_inited = 0; 184612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Use WXOR for self-testing */ 184712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!ppc440spe_r6_tchan) 184812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_r6_tchan = ppc440spe_chan; 184912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 185012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 185112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_start_null_xor(ppc440spe_chan); 185212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 185312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 185412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 185512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 185612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->needs_unmap = 1; 185712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 185812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 185912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return (i > 0) ? i : -ENOMEM; 186012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 186112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 186212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 186312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_rxor_set_region_data - 186412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 186512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc, 186612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 xor_arg_no, u32 mask) 186712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 186812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 186912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 187012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->ops[xor_arg_no].h |= mask; 187112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 187212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 187312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 187412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_rxor_set_src - 187512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 187612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc, 187712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 xor_arg_no, dma_addr_t addr) 187812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 187912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 188012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 188112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE; 188212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->ops[xor_arg_no].l = addr; 188312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 188412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 188512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 188612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_rxor_set_mult - 188712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 188812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc, 188912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 xor_arg_no, u8 idx, u8 mult) 189012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 189112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 189212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 189312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8); 189412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 189512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 189612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 189712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold 189812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * has been achieved 189912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 190012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan) 190112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 190212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n", 190312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id, chan->pending); 190412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 190512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) { 190612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->pending = 0; 190712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_append(chan); 190812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 190912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 191012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 191112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 191212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_tx_submit - submit new descriptor group to the channel 191312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (it's not necessary that descriptors will be submitted to the h/w 191412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * chains too right now) 191512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 191612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx) 191712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 191812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc; 191912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan); 192012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail; 192112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 192212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slots_per_op; 192312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t cookie; 192412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 192512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = tx_to_ppc440spe_adma_slot(tx); 192612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 192712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 192812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = group_start->slot_cnt; 192912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = group_start->slots_per_op; 193012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 193112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&chan->lock); 1932884485e1f12dcd39390f042e772cdbefc9ebb750Russell King - ARM Linux cookie = dma_cookie_assign(tx); 193312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 193412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(list_empty(&chan->chain))) { 193512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* first peer */ 193612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_splice_init(&sw_desc->group_list, &chan->chain); 193712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan_first_cdb[chan->device->id] = group_start; 193812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 193912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* isn't first peer, bind CDBs to chain */ 194012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin old_chain_tail = list_entry(chan->chain.prev, 194112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 194212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 194312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_splice_init(&sw_desc->group_list, 194412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &old_chain_tail->chain_node); 194512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* fix up the hardware chain */ 194612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_link(chan, old_chain_tail, group_start); 194712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 194812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 194912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* increment the pending count by the number of operations */ 195012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->pending += slot_cnt / slots_per_op; 195112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_check_threshold(chan); 195212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 195312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 195412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, 195512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n", 195612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id, __func__, 195712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.cookie, sw_desc->idx, sw_desc); 195812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 195912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return cookie; 196012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 196112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 196212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 196312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation 196412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 196512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt( 196612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, unsigned long flags) 196712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 196812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 196912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 197012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op; 197112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 197212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 197312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 197412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 197512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id, 197612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 197712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 197812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 197912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = slots_per_op = 1; 198012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 198112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op); 198212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 198312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 198412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan); 198512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start->unmap_len = 0; 198612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 198712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 198812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 198912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 199012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 199112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 199212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 199312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 199412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation 199512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 199612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy( 199712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t dma_dest, 199812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t dma_src, size_t len, unsigned long flags) 199912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 200012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 200112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 200212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op; 200312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 200412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 200512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 200612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!len)) 200712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 200812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 2009427cdf19b97e509e21e5d347e18d8b0b34723dfcColy Li BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT); 201012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 201112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 201212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 201312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 201412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s len: %u int_en %d\n", 201512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, __func__, len, 201612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags & DMA_PREP_INTERRUPT ? 1 : 0); 201712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = slots_per_op = 1; 201812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 201912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op); 202012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 202112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 202212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_memcpy(group_start, flags); 202312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_set_dest(group_start, dma_dest, 0); 202412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0); 202512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len); 202612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->unmap_len = len; 202712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 202812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 202912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 203012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 203112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 203212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 203312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 203412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 203512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation 203612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 203712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor( 203812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t dma_dest, 203912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dma_src, u32 src_cnt, size_t len, 204012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long flags) 204112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 204212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 204312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 204412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op; 204512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 204612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 204712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 204812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id, 204912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_dest, dma_src, src_cnt)); 205012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!len)) 205112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 2052427cdf19b97e509e21e5d347e18d8b0b34723dfcColy Li BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 205312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 205412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 205512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n", 205612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, __func__, src_cnt, len, 205712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags & DMA_PREP_INTERRUPT ? 1 : 0); 205812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 205912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 206012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op); 206112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 206212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op); 206312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 206412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 206512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_xor(group_start, src_cnt, flags); 206612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_set_dest(group_start, dma_dest, 0); 206712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (src_cnt--) 206812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_memcpy_xor_set_src(group_start, 206912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_src[src_cnt], src_cnt); 207012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len); 207112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->unmap_len = len; 207212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 207312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 207412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 207512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 207612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 207712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 207812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 207912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline void 208012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc, 208112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt); 208212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor); 208312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 208412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 208512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_init_dma2rxor_slot - 208612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 208712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_init_dma2rxor_slot( 208812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 208912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *src, int src_cnt) 209012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 209112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 209212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 209312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* initialize CDB */ 209412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) { 209512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i, 209612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt, (u32)src[i]); 209712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 209812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 209912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 210012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 210112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma01_prep_mult - 210212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for Q operation where destination is also the source 210312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 210412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult( 210512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan, 210612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 210712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf, size_t len, unsigned long flags) 210812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 210912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL; 211012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 211112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 211212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 211312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_WXOR, &op); 211412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = 2; 211512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 211612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 211712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 211812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* use WXOR, each descriptor occupies one slot */ 211912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 212012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 212112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 212212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 212312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 212412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 212512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 212612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bits(op, &sw_desc->flags); 212712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->src_cnt = src_cnt; 212812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt = dst_cnt; 212912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* First descriptor, zero data in the destination and copy it 213012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * to q page using MULTICAST transfer. 213112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 213212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&sw_desc->group_list, 213312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 213412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 213512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 213612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 213712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 213812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 213912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 214012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 214112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 214212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MULTICAST; 214312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 214412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 214512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, dst[0], 0); 214612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1); 214712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 214812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src[0]); 214912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 215012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 215112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 215212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 215312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Second descriptor, multiply data from the q page 215412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * and store the result in real destination. 215512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 215612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 215712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 215812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 215912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 216012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 216112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 216212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 216312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 216412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 216512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 216612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 216712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 216812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 216912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB, dst[1]); 217012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 217112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, dst[0], 0); 217212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 217312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 217412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1, scf[0]); 217512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 217612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 217712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 217812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 217912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 218012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 218112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 218212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc; 218312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 218412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 218512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 218612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma01_prep_sum_product - 218712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also 218812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the source. 218912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 219012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product( 219112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan, 219212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dst, dma_addr_t *src, int src_cnt, 219312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf, size_t len, unsigned long flags) 219412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 219512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL; 219612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 219712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 219812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 219912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_WXOR, &op); 220012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = 3; 220112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 220212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 220312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 220412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR, each descriptor occupies one slot */ 220512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 220612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 220712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 220812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 220912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 221012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 221112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 221212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bits(op, &sw_desc->flags); 221312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->src_cnt = src_cnt; 221412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt = 1; 221512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 1st descriptor, src[1] data to q page and zero destination */ 221612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&sw_desc->group_list, 221712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 221812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 221912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 222012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 222112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 222212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 222312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 222412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 222512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MULTICAST; 222612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 222712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 222812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *dst, 0); 222912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 0, 223012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->qdest, 1); 223112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 223212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src[1]); 223312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 223412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 223512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 223612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 2nd descriptor, multiply src[1] data and store the 223712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * result in destination */ 223812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 223912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 224012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 224112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 224212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 224312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 224412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 224512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 224612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 224712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 224812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 224912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 225012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 225112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 225212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 225312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 225412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->qdest); 225512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 225612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *dst, 0); 225712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 225812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1, scf[1]); 225912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 226012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 226112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 226212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 226312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 3rd descriptor, multiply src[0] data and xor it 226412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * with destination 226512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 226612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 226712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 226812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 226912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 227012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 227112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 227212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 227312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 227412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 227512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 227612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 227712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 227812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 227912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src[0]); 228012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 228112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *dst, 0); 228212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 228312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1, scf[0]); 228412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 228512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 228612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 228712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 228812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 228912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 229012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 229112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc; 229212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 229312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 229412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq( 229512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan, 229612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 229712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf, size_t len, unsigned long flags) 229812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 229912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 230012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter; 230112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 230212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned char mult = 1; 230312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 230412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n", 230512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dst_cnt, src_cnt, len); 230612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* select operations WXOR/RXOR depending on the 230712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * source addresses of operators and the number 230812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * of destinations (RXOR support only Q-parity calculations) 230912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 231012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_WXOR, &op); 231112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) { 231212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* no active RXOR; 231312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * do RXOR if: 231412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - there are more than 1 source, 231512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - len is aligned on 512-byte boundary, 231612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - source addresses fit to one of 4 possible regions. 231712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 231812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt > 1 && 231912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin !(len & MQ0_CF2H_RXOR_BS_MASK) && 232012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (src[0] + len) == src[1]) { 232112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* may do RXOR R1 R2 */ 232212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR, &op); 232312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt != 2) { 232412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* may try to enhance region of RXOR */ 232512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((src[1] + len) == src[2]) { 232612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 R3 */ 232712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR123, 232812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &op); 232912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if ((src[1] + len * 2) == src[2]) { 233012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 R4 */ 233112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR124, &op); 233212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if ((src[1] + len * 3) == src[2]) { 233312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 R5 */ 233412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR125, 233512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &op); 233612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 233712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 */ 233812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR12, 233912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &op); 234012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 234112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 234212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 */ 234312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR12, &op); 234412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 234512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 234612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 234712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_RXOR, &op)) { 234812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* can not do this operation with RXOR */ 234912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_RXOR_RUN, 235012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &ppc440spe_rxor_state); 235112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 235212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* can do; set block size right now */ 235312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_rxor_block_size(len); 235412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 235512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 235612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 235712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Number of necessary slots depends on operation type selected */ 235812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_RXOR, &op)) { 235912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is a WXOR only chain. Need descriptors for each 236012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * source to GF-XOR them with WXOR, and need descriptors 236112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for each destination to zero them with WXOR 236212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 236312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = src_cnt; 236412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 236512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_P) { 236612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt++; 236712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_P, &op); 236812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 236912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_Q) { 237012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt++; 237112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_Q, &op); 237212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 237312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 237412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Need 1/2 descriptor for RXOR operation, and 237512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * need (src_cnt - (2 or 3)) for WXOR of sources 237612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * remained (if any) 237712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 237812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = dst_cnt; 237912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 238012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_P) 238112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_P, &op); 238212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_Q) 238312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_Q, &op); 238412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 238512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_RXOR12, &op)) 238612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt += src_cnt - 2; 238712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 238812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt += src_cnt - 3; 238912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 239012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Thus we have either RXOR only chain or 239112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * mixed RXOR/WXOR 239212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 239312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slot_cnt == dst_cnt) 239412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR only chain */ 239512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_WXOR, &op); 239612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 239712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 239812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 239912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* for both RXOR/WXOR each descriptor occupies one slot */ 240012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 240112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 240212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt, 240312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags, op); 240412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 240512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* setup dst/src/mult */ 240612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n", 240712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dst[0], dst[1]); 240812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_dest(sw_desc, dst, flags); 240912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (src_cnt--) { 241012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt], 241112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt); 241212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 241312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* NOTE: "Multi = 0 is equivalent to = 1" as it 241412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf 241512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * doesn't work for RXOR with DMA0/1! Instead, multi=0 241612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * leads to zeroing source data after RXOR. 241712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * So, for P case set-up mult=1 explicitly. 241812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 241912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 242012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult = scf[src_cnt]; 242112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src_mult(sw_desc, 242212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult, src_cnt, dst_cnt - 1); 242312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 242412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 242512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup byte count foreach slot just allocated */ 242612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 242712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, 242812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 242912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, 243012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan, len); 243112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 243212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 243312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 243412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 243512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 243612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc; 243712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 243812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 243912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq( 244012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan, 244112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 244212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf, size_t len, unsigned long flags) 244312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 244412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, descs_per_op; 244512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter; 244612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 244712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned char mult = 1; 244812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 244912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!dst_cnt); 245012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n", 245112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dst_cnt, src_cnt, len);*/ 245212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 245312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 245412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len); 245512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (descs_per_op < 0) { 245612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 245712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 245812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 245912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 246012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* depending on number of sources we have 1 or 2 RXOR chains */ 246112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = descs_per_op * dst_cnt; 246212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 246312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 246412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 246512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op = slot_cnt; 246612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 246712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 246812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt, 246912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin --op ? 0 : flags); 247012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 247112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len); 247212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 247312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 247412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_init_rxor_cursor(&(iter->rxor_cursor)); 247512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->rxor_cursor.len = len; 247612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->descs_per_op = descs_per_op; 247712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 247812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op = 0; 247912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 248012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op++; 248112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (op % descs_per_op == 0) 248212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_init_dma2rxor_slot(iter, src, 248312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt); 248412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(!list_is_last(&iter->chain_node, 248512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list))) { 248612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 248712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = 248812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_entry(iter->chain_node.next, 248912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 249012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 249112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_xor_set_link(iter, iter->hw_next); 249212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 249312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is the last descriptor. */ 249412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 249512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 249612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 249712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 249812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* fixup head descriptor */ 249912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt = dst_cnt; 250012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_P) 250112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_P, &sw_desc->flags); 250212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_Q) 250312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags); 250412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 250512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* setup dst/src/mult */ 250612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_dest(sw_desc, dst, flags); 250712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 250812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (src_cnt--) { 250912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* handle descriptors (if dst_cnt == 2) inside 251012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the ppc440spe_adma_pq_set_srcxxx() functions 251112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 251212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt], 251312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt); 251412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 251512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult = scf[src_cnt]; 251612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src_mult(sw_desc, 251712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult, src_cnt, dst_cnt - 1); 251812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 251912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 252012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 252112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_rxor_block_size(len); 252212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc; 252312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 252412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 252512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 252612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation 252712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 252812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq( 252912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, 253012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt, const unsigned char *scf, 253112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size_t len, unsigned long flags) 253212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 253312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 253412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL; 253512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int dst_cnt = 0; 253612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 253712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 253812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 253912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id, 254012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst, src, src_cnt)); 254112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!len); 2542427cdf19b97e509e21e5d347e18d8b0b34723dfcColy Li BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 254312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!src_cnt); 254412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 254512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt == 1 && dst[1] == src[0]) { 254612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t dest[2]; 254712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 254812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* dst[1] is real destination (Q) */ 254912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dest[0] = dst[1]; 255012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is the page to multicast source data to */ 255112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dest[1] = ppc440spe_chan->qdest; 255212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan, 255312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dest, 2, src, src_cnt, scf, len, flags); 255412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 255512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 255612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 255712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt == 2 && dst[1] == src[1]) { 255812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan, 255912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &dst[1], src, 2, scf, len, flags); 256012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 256112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 256212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 256312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(flags & DMA_PREP_PQ_DISABLE_P)) { 256412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!dst[0]); 256512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst_cnt++; 256612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags |= DMA_PREP_ZERO_P; 256712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 256812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 256912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(flags & DMA_PREP_PQ_DISABLE_Q)) { 257012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!dst[1]); 257112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst_cnt++; 257212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags |= DMA_PREP_ZERO_Q; 257312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 257412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 257512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!dst_cnt); 257612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 257712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 257812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n", 257912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, __func__, src_cnt, len, 258012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags & DMA_PREP_INTERRUPT ? 1 : 0); 258112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 258212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (ppc440spe_chan->device->id) { 258312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 258412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 258512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan, 258612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst, dst_cnt, src, src_cnt, scf, 258712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len, flags); 258812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 258912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 259012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 259112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan, 259212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst, dst_cnt, src, src_cnt, scf, 259312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len, flags); 259412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 259512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 259612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 259712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 259812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 259912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 260012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 260112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for 260212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * a PQ_ZERO_SUM operation 260312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 260412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum( 260512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, 260612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt, const unsigned char *scf, size_t len, 260712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin enum sum_check_flags *pqres, unsigned long flags) 260812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 260912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 261012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *iter; 261112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t pdest, qdest; 261212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op, idst, dst_cnt; 261312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 261412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 261512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 261612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_PQ_DISABLE_P) 261712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pdest = 0; 261812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 261912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pdest = pq[0]; 262012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 262112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_PQ_DISABLE_Q) 262212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qdest = 0; 262312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 262412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qdest = pq[1]; 262512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 262612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id, 262712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src, src_cnt, scf)); 262812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 262912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Always use WXOR for P/Q calculations (two destinations). 263012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Need 1 or 2 extra slots to verify results are zero. 263112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 263212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idst = dst_cnt = (pdest && qdest) ? 2 : 1; 263312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 263412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* One additional slot per destination to clone P/Q 263512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * before calculation (we have to preserve destinations). 263612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 263712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = src_cnt + dst_cnt * 2; 263812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = 1; 263912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 264012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 264112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 264212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op); 264312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 264412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt); 264512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 264612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup byte count for each slot just allocated */ 264712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 264812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 264912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 265012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len); 265112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 265212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 265312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 265412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (pdest) { 265512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 265612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 265712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 265812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = sw_desc->group_head; 265912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 266012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 266112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 266212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 266312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 266412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 266512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 266612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->src_cnt = 0; 266712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->dst_cnt = 0; 266812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 0, 266912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->pdest, 0); 267012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest); 267112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 267212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len); 267312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = 0; 267412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* override pdest to preserve original P */ 267512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pdest = ppc440spe_chan->pdest; 267612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 267712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (qdest) { 267812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 267912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 268012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 268112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&sw_desc->group_list, 268212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 268312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 268412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 268512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 268612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (pdest) { 268712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 268812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 268912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 269012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 269112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 269212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 269312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 269412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 269512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 269612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 269712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 269812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->src_cnt = 0; 269912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->dst_cnt = 0; 270012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 0, 270112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->qdest, 0); 270212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest); 270312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 270412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len); 270512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = 0; 270612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* override qdest to preserve original Q */ 270712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qdest = ppc440spe_chan->qdest; 270812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 270912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 271012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup destinations for P/Q ops */ 271112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest); 271212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 271312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup zero QWORDs into DCHECK CDBs */ 271412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idst = dst_cnt; 271512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_reverse(iter, &sw_desc->group_list, 271612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 271712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 271812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * The last CDB corresponds to Q-parity check, 271912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the one before last CDB corresponds 272012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * P-parity check 272112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 272212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (idst == DMA_DEST_MAX_NUM) { 272312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (idst == dst_cnt) { 272412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_QCHECK, 272512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags); 272612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 272712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_PCHECK, 272812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags); 272912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 273012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 273112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (qdest) { 273212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_QCHECK, 273312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags); 273412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 273512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_PCHECK, 273612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags); 273712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 273812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 273912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->xor_check_result = pqres; 274012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 274112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 274212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * set it to zero, if check fail then result will 274312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * be updated 274412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 274512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *iter->xor_check_result = 0; 274612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dcheck(iter, ppc440spe_chan, 274712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_qword); 274812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 274912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(--dst_cnt)) 275012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 275112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 275212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 275312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup sources and mults for P/Q ops */ 275412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_continue_reverse(iter, &sw_desc->group_list, 275512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 275612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 275712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 mult_dst; 275812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 275912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 276012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 276112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB, 276212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src[src_cnt - 1]); 276312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (qdest) { 276412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 : 276512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1; 276612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, 276712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_MULT1_OFF, 276812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst, 276912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin scf[src_cnt - 1]); 277012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 277112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(--src_cnt)) 277212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 277312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 277412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 277512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 277612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 277712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 277812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 277912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 278012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for 278112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * XOR ZERO_SUM operation 278212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 278312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum( 278412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt, 278512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size_t len, enum sum_check_flags *result, unsigned long flags) 278612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 278712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_async_tx_descriptor *tx; 278812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t pq[2]; 278912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 279012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* validate P, disable Q */ 279112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pq[0] = src[0]; 279212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pq[1] = 0; 279312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags |= DMA_PREP_PQ_DISABLE_Q; 279412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 279512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1], 279612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt - 1, 0, len, 279712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin result, flags); 279812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return tx; 279912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 280012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 280112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 280212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_set_dest - set destination address into descriptor 280312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 280412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc, 280512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index) 280612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 280712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 280812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 280912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(index >= sw_desc->dst_cnt); 281012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 281112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 281212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 281312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 281412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 281512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 281612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* to do: support transfers lengths > 281712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT 281812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 281912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(sw_desc->group_head, 282012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan, 0, addr, index); 282112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 282212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 282312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_get_group_entry(sw_desc, index); 282412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(sw_desc, 282512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan, 0, addr, index); 282612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 282712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 282812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 282912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 283012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter, 283112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, dma_addr_t addr) 283212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 283312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* To clear destinations update the descriptor 283412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (P or Q depending on index) as follows: 283512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * addr is destination (0 corresponds to SG2): 283612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 283712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0); 283812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 283912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* ... and the addr is source: */ 284012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr); 284112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 284212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* addr is always SG2 then the mult is always DST1 */ 284312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 284412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1, 1); 284512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 284612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 284712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 284812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_pq_set_dest - set destination address into descriptor 284912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for the PQXOR operation 285012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 285112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc, 285212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *addrs, unsigned long flags) 285312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 285412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 285512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 285612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t paddr, qaddr; 285712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr = 0, ppath, qpath; 285812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index = 0, i; 285912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 286012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 286112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 286212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_PQ_DISABLE_P) 286312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr = 0; 286412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 286512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr = addrs[0]; 286612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 286712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_PQ_DISABLE_Q) 286812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qaddr = 0; 286912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 287012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qaddr = addrs[1]; 287112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 287212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!paddr || !qaddr) 287312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr = paddr ? paddr : qaddr; 287412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 287512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 287612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 287712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 287812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* walk through the WXOR source list and set P/Q-destinations 287912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for each slot: 288012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 288112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 288212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is WXOR-only chain; may have 1/2 zero descs */ 288312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 288412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index++; 288512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 288612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index++; 288712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 288812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, index); 288912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (addr) { 289012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* one destination */ 289112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, 289212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list, chain_node) 289312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 289412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, addr, 0); 289512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 289612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* two destinations */ 289712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, 289812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list, chain_node) { 289912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 290012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, paddr, 0); 290112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 290212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, qaddr, 1); 290312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 290412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 290512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 290612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index) { 290712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* To clear destinations update the descriptor 290812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (1st,2nd, or both depending on flags) 290912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 291012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index = 0; 291112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, 291212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) { 291312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry( 291412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc, index++); 291512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_zero_op(iter, chan, 291612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr); 291712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 291812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 291912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, 292012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) { 292112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry( 292212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc, index++); 292312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_zero_op(iter, chan, 292412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qaddr); 292512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 292612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 292712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return; 292812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 292912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 293012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is RXOR-only or RXOR/WXOR mixed chain */ 293112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 293212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* If we want to include destination into calculations, 293312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * then make dest addresses cued with mult=1 (XOR). 293412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 293512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ? 293612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB : 293712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE | 293812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << DMA_CUED_MULT1_OFF); 293912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ? 294012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB : 294112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE | 294212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << DMA_CUED_MULT1_OFF); 294312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 294412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup destination(s) in RXOR slot(s) */ 294512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, index++); 294612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 294712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr ? ppath : qpath, 294812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr ? paddr : qaddr, 0); 294912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addr) { 295012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* two destinations */ 295112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 295212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index++); 295312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 295412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qpath, qaddr, 0); 295512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 295612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 295712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) { 295812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup destination(s) in remaining WXOR 295912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * slots 296012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 296112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 296212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index); 296312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (addr) { 296412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* one destination */ 296512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, 296612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list, 296712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) 296812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr( 296912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter, chan, 297012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, 297112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr, 0); 297212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 297312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 297412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* two destinations */ 297512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, 297612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list, 297712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 297812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr( 297912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter, chan, 298012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, 298112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr, 0); 298212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr( 298312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter, chan, 298412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, 298512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qaddr, 1); 298612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 298712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 298812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 298912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 299012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 299112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 299212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 299312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 299412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA2 descriptors have only 1 destination, so there are 299512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * two chains - one for each dest. 299612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * If we want to include destination into calculations, 299712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * then make dest addresses cued with mult=1 (XOR). 299812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 299912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ? 300012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB : 300112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE | 300212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << DMA_CUED_MULT1_OFF); 300312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 300412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ? 300512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB : 300612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE | 300712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << DMA_CUED_MULT1_OFF); 300812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 300912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 0); 301012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < sw_desc->descs_per_op; i++) { 301112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 301212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr ? ppath : qpath, 301312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr ? paddr : qaddr, 0); 301412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 301512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 301612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 301712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 301812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 301912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addr) { 302012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Two destinations; setup Q here */ 302112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 302212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->descs_per_op); 302312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < sw_desc->descs_per_op; i++) { 302412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, 302512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan, qpath, qaddr, 0); 302612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 302712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 302812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 302912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 303012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 303112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 303212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 303312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 303412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 303512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 303612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 303712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor 303812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for the PQ_ZERO_SUM operation 303912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 304012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pqzero_sum_set_dest( 304112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, 304212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t paddr, dma_addr_t qaddr) 304312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 304412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter, *end; 304512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 304612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr = 0; 304712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int idx; 304812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 304912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 305012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 305112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* walk through the WXOR source list and set P/Q-destinations 305212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for each slot 305312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 305412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idx = (paddr && qaddr) ? 2 : 1; 305512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set end */ 305612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_reverse(end, &sw_desc->group_list, 305712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 305812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(--idx)) 305912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 306012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 306112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set start */ 306212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idx = (paddr && qaddr) ? 2 : 1; 306312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, idx); 306412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 306512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (paddr && qaddr) { 306612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* two destinations */ 306712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &sw_desc->group_list, 306812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 306912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(iter == end)) 307012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 307112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 307212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, paddr, 0); 307312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 307412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, qaddr, 1); 307512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 307612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 307712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* one destination */ 307812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr = paddr ? paddr : qaddr; 307912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &sw_desc->group_list, 308012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 308112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(iter == end)) 308212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 308312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 308412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, addr, 0); 308512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 308612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 308712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 308812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* The remaining descriptors are DATACHECK. These have no need in 308912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * destination. Actually, these destinations are used there 309012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * as sources for check operation. So, set addr as source. 309112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 309212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr); 309312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 309412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addr) { 309512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin end = list_entry(end->chain_node.next, 309612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 309712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr); 309812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 309912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 310012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 310112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 310212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor 310312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 310412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline void ppc440spe_desc_set_xor_src_cnt( 310512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 310612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt) 310712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 310812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *hw_desc = desc->hw_desc; 310912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 311012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc &= ~XOR_CDCR_OAC_MSK; 311112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc |= src_cnt; 311212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 311312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 311412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 311512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_pq_set_src - set source address into descriptor 311612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 311712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc, 311812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index) 311912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 312012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 312112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t haddr = 0; 312212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter = NULL; 312312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 312412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 312512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 312612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 312712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 312812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 312912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain 313012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 313112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 313212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR-only or RXOR/WXOR operation */ 313312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int iskip = test_bit(PPC440SPE_DESC_RXOR12, 313412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags) ? 2 : 3; 313512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 313612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == 0) { 313712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 1st slot (RXOR) */ 313812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* setup sources region (R1-2-3, R1-2-4, 313912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * or R1-2-5) 314012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 314112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_RXOR12, 314212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) 314312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_RXOR12 << 314412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 314512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else if (test_bit(PPC440SPE_DESC_RXOR123, 314612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) 314712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_RXOR123 << 314812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 314912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else if (test_bit(PPC440SPE_DESC_RXOR124, 315012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) 315112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_RXOR124 << 315212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 315312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else if (test_bit(PPC440SPE_DESC_RXOR125, 315412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) 315512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_RXOR125 << 315612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 315712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 315812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 315912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr |= DMA_CUED_XOR_BASE; 316012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 0); 316112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (index < iskip) { 316212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 1st slot (RXOR) 316312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * shall actually set source address only once 316412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * instead of first <iskip> 316512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 316612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = NULL; 316712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 316812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 2nd/3d and next slots (WXOR); 316912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * skip first slot with RXOR 317012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 317112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_CUED_XOR_HB; 317212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 317312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index - iskip + sw_desc->dst_cnt); 317412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 317512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 317612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int znum = 0; 317712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 317812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR-only operation; skip first slots with 317912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * zeroing destinations 318012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 318112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 318212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin znum++; 318312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 318412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin znum++; 318512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 318612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_CUED_XOR_HB; 318712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 318812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index + znum); 318912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 319012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 319112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(iter)) { 319212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr); 319312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 319412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!index && 319512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) && 319612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt == 2) { 319712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* if we have two destinations for RXOR, then 319812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * setup source in the second descr too 319912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 320012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 1); 320112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 320212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr, addr); 320312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 320412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 320512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 320612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 320712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 320812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA2 may do Biskup */ 320912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = sw_desc->group_head; 321012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->dst_cnt == 2) { 321112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* both P & Q calculations required; set P src here */ 321212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_set_src(iter, index, addr); 321312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 321412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is for Q */ 321512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 321612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->descs_per_op); 321712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 321812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_set_src(iter, index, addr); 321912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 322012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 322112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 322212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 322312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 322412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor 322512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 322612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_memcpy_xor_set_src( 322712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, 322812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index) 322912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 323012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 323112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 323212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 323312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = sw_desc->group_head; 323412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 323512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(sw_desc)) 323612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr); 323712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 323812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 323912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 324012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_inc_addr - 324112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 324212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_inc_addr( 324312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 324412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_rxor *cursor, int index, int src_cnt) 324512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 324612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count++; 324712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt - 1) { 324812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count); 324912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cursor->addr_count == XOR_MAX_OPS) { 325012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count); 325112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count = 0; 325212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->desc_count++; 325312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 325412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 325512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 325612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 325712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB 325812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 325912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_dma2rxor_prep_src( 326012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *hdesc, 326112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_rxor *cursor, int index, 326212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, u32 addr) 326312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 326412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int rval = 0; 326512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 sign; 326612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc = hdesc; 326712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 326812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 326912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < cursor->desc_count; i++) { 327012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc = list_entry(hdesc->chain_node.next, 327112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 327212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 327312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 327412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 327512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (cursor->state) { 327612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 0: 327712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (addr == cursor->addrl + cursor->len) { 327812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct RXOR */ 327912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 1; 328012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count++; 328112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 328212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 328312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 328412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR12 << DMA_CUED_REGION_OFF); 328512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 328612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 328712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 328812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cursor->addrl == addr + cursor->len) { 328912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse RXOR */ 329012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 1; 329112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count++; 329212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(cursor->addr_count, &desc->reverse_flags[0]); 329312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 329412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 329512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 329612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR12 << DMA_CUED_REGION_OFF); 329712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 329812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 329912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 330012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 330112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "Cannot build " 330212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "DMA2 RXOR command block.\n"); 330312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 330412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 330512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 330612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 1: 330712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sign = test_bit(cursor->addr_count, 330812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->reverse_flags) 330912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ? -1 : 1; 331012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-2 || (sign == -1 331112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin && addr != cursor->addrl - 2*cursor->len)) { 331212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 0; 331312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 1; 331412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addrl = addr; 331512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 331612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 331712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR12 << DMA_CUED_REGION_OFF); 331812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 331912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 332012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (addr == cursor->addrl + 2*sign*cursor->len) { 332112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 2; 332212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 0; 332312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 332412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 332512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR123 << DMA_CUED_REGION_OFF); 332612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 332712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 332812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 332912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 333012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (addr == cursor->addrl + 3*cursor->len) { 333112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 2; 333212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 0; 333312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 333412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 333512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR124 << DMA_CUED_REGION_OFF); 333612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 333712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 333812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 333912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 334012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (addr == cursor->addrl + 4*cursor->len) { 334112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 2; 334212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 0; 334312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 334412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 334512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR125 << DMA_CUED_REGION_OFF); 334612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 334712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 334812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 334912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 335012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 335112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 0; 335212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 1; 335312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addrl = addr; 335412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 335512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 335612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR12 << DMA_CUED_REGION_OFF); 335712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 335812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 335912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 336012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 336112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 2: 336212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 0; 336312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addrl = addr; 336412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count++; 336512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index) { 336612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 336712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 336812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 336912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 337012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 337112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 337212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return rval; 337312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 337412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 337512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 337612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that 337712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call 337812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 337912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_set_src( 338012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 338112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index, dma_addr_t addr) 338212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 338312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 338412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int k = 0, op = 0, lop = 0; 338512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 338612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* get the RXOR operand which corresponds to index addr */ 338712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (op <= index) { 338812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin lop = op; 338912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (k == XOR_MAX_OPS) { 339012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin k = 0; 339112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc = list_entry(desc->chain_node.next, 339212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 339312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb = desc->hw_desc; 339412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 339512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 339612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) == 339712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (DMA_RXOR12 << DMA_CUED_REGION_OFF)) 339812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op += 2; 339912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 340012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op += 3; 340112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 340212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 340312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(k < 1); 340412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 340512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(k-1, desc->reverse_flags)) { 340612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse operand order; put last op in RXOR group */ 340712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == op - 1) 340812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_src(desc, k - 1, addr); 340912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 341012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct operand order; put first op in RXOR group */ 341112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == lop) 341212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_src(desc, k - 1, addr); 341312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 341412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 341512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 341612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 341712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that 341812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call 341912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 342012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_set_mult( 342112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 342212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index, u8 mult) 342312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 342412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 342512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int k = 0, op = 0, lop = 0; 342612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 342712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* get the RXOR operand which corresponds to index mult */ 342812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (op <= index) { 342912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin lop = op; 343012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (k == XOR_MAX_OPS) { 343112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin k = 0; 343212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc = list_entry(desc->chain_node.next, 343312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 343412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 343512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb = desc->hw_desc; 343612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 343712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 343812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) == 343912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (DMA_RXOR12 << DMA_CUED_REGION_OFF)) 344012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op += 2; 344112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 344212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op += 3; 344312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 344412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 344512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(k < 1); 344612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(k-1, desc->reverse_flags)) { 344712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse order */ 344812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult); 344912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 345012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct order */ 345112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult); 345212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 345312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 345412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 345512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 345612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_init_rxor_cursor - 345712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 345812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor) 345912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 346012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(cursor, 0, sizeof(struct ppc440spe_rxor)); 346112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 2; 346212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 346312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 346412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 346512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into 346612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor for the PQXOR operation 346712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 346812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pq_set_src_mult( 346912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, 347012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned char mult, int index, int dst_pos) 347112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 347212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 347312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 mult_idx, mult_dst; 347412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL; 347512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 347612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 347712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 347812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 347912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 348012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 348112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 348212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int region = test_bit(PPC440SPE_DESC_RXOR12, 348312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags) ? 2 : 3; 348412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 348512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index < region) { 348612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR multipliers */ 348712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 348812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt - 1); 348912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc->dst_cnt == 2) 349012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter1 = ppc440spe_get_group_entry( 349112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc, 0); 349212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 349312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx = DMA_CUED_MULT1_OFF + (index << 3); 349412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst = DMA_CDB_SG_SRC; 349512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 349612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR multiplier */ 349712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 349812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index - region + 349912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt); 350012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx = DMA_CUED_MULT1_OFF; 350112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst = dst_pos ? DMA_CDB_SG_DST2 : 350212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1; 350312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 350412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 350512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int znum = 0; 350612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 350712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR-only; 350812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * skip first slots with destinations (if ZERO_DST has 350912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * place) 351012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 351112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 351212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin znum++; 351312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 351412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin znum++; 351512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 351612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, index + znum); 351712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx = DMA_CUED_MULT1_OFF; 351812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1; 351912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 352012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 352112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(iter)) { 352212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, 352312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx, mult_dst, mult); 352412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 352512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(iter1)) { 352612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* if we have two destinations for RXOR, then 352712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * we've just set Q mult. Set-up P now. 352812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 352912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter1, chan, 353012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx, mult_dst, 1); 353112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 353212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 353312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 353412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 353512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 353612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 353712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = sw_desc->group_head; 353812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc->dst_cnt == 2) { 353912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* both P & Q calculations required; set P mult here */ 354012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_set_mult(iter, index, 1); 354112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 354212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* and then set Q mult */ 354312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 354412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->descs_per_op); 354512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 354612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_set_mult(iter, index, mult); 354712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 354812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 354912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 355012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 355112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 355212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_free_chan_resources - free the resources allocated 355312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 355412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_free_chan_resources(struct dma_chan *chan) 355512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 355612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 355712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter, *_iter; 355812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int in_use_descs = 0; 355912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 356012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 356112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_slot_cleanup(ppc440spe_chan); 356212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 356312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 356412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain, 356512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 356612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin in_use_descs++; 356712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&iter->chain_node); 356812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 356912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe_reverse(iter, _iter, 357012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &ppc440spe_chan->all_slots, slot_node) { 357112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&iter->slot_node); 357212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(iter); 357312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->slots_allocated--; 357412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 357512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->last_used = NULL; 357612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 357712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 357812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d %s slots_allocated %d\n", 357912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, 358012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, ppc440spe_chan->slots_allocated); 358112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 358212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 358312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* one is ok since we left it on there on purpose */ 358412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (in_use_descs > 1) 358512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n", 358612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin in_use_descs - 1); 358712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 358812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 358912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 35900793448187643b50af89d36b08470baf45a3cab4Linus Walleij * ppc440spe_adma_tx_status - poll the status of an ADMA transaction 359112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @chan: ADMA channel handle 359212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @cookie: ADMA transaction identifier 35930793448187643b50af89d36b08470baf45a3cab4Linus Walleij * @txstate: a holder for the current state of the channel 359412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 35950793448187643b50af89d36b08470baf45a3cab4Linus Walleijstatic enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan, 35960793448187643b50af89d36b08470baf45a3cab4Linus Walleij dma_cookie_t cookie, struct dma_tx_state *txstate) 359712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 359812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 359912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin enum dma_status ret; 360012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 360112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 360296a2af41c78b1fbb1f567a3486bdc63f7b31c5fdRussell King - ARM Linux ret = dma_cookie_status(chan, cookie, txstate); 36035738992b078140267f6637f9f929ec7581f6213eVinod Koul if (ret == DMA_COMPLETE) 360412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 360512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 360612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_slot_cleanup(ppc440spe_chan); 360712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 360896a2af41c78b1fbb1f567a3486bdc63f7b31c5fdRussell King - ARM Linux return dma_cookie_status(chan, cookie, txstate); 360912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 361012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 361112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 361212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_eot_handler - end of transfer interrupt handler 361312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 361412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data) 361512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 361612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan = data; 361712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 361812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, 361912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s\n", chan->device->id, __func__); 362012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 362112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_schedule(&chan->irq_tasklet); 362212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_device_clear_eot_status(chan); 362312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 362412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return IRQ_HANDLED; 362512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 362612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 362712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 362812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_err_handler - DMA error interrupt handler; 362912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * do the same things as a eot handler 363012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 363112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic irqreturn_t ppc440spe_adma_err_handler(int irq, void *data) 363212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 363312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan = data; 363412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 363512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, 363612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s\n", chan->device->id, __func__); 363712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 363812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_schedule(&chan->irq_tasklet); 363912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_device_clear_eot_status(chan); 364012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 364112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return IRQ_HANDLED; 364212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 364312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 364412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 364512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_test_callback - called when test operation has been done 364612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 364712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_test_callback(void *unused) 364812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 364912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin complete(&ppc440spe_r6_test_comp); 365012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 365112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 365212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 365312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w 365412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 365512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_issue_pending(struct dma_chan *chan) 365612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 365712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 365812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 365912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 366012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 366112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id, 366212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, ppc440spe_chan->pending); 366312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 366412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_chan->pending) { 366512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->pending = 0; 366612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_append(ppc440spe_chan); 366712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 366812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 366912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 367012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 367112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines 367212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * use FIFOs (as opposite to chains used in XOR) so this is a XOR 367312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * specific operation) 367412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 367512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan) 367612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 367712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 367812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t cookie; 367912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op; 368012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 368112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, 368212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s\n", chan->device->id, __func__); 368312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 368412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&chan->lock); 368512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op); 368612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op); 368712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 368812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 368912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_splice_init(&sw_desc->group_list, &chan->chain); 369012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin async_tx_ack(&sw_desc->async_tx); 369112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_null_xor(group_start); 369212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 36932a926e46022ad7a03e0ac167d8c2b0d88c12c5a8Russell King - ARM Linux cookie = dma_cookie_assign(&sw_desc->async_tx); 369412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 369512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* initialize the completed cookie to be less than 369612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the most recently used cookie 369712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 36984d4e58de32a192fea65ab84509d17d199bd291c8Russell King - ARM Linux chan->common.completed_cookie = cookie - 1; 369912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 370012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* channel should not be busy */ 370112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(ppc440spe_chan_is_busy(chan)); 370212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 370312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set the descriptor address */ 370412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc); 370512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 370612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* run the descriptor */ 370712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_run(chan); 370812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 370912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "ppc440spe adma%d" 371012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " failed to allocate null descriptor\n", 371112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id); 371212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 371312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 371412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 371512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 371612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully. 371712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * For this we just perform one WXOR operation with the same source 371812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * and destination addresses, the GF-multiplier is 1; so if RAID-6 371912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * capabilities are enabled then we'll get src/dst filled with zero. 372012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 372112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan) 372212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 372312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *iter; 372412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct page *pg; 372512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin char *a; 372612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t dma_addr, addrs[2]; 372712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 372812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int rval = 0; 372912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 373012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_WXOR, &op); 373112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 373212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pg = alloc_page(GFP_KERNEL); 373312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!pg) 373412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENOMEM; 373512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 373612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&chan->lock); 373712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1); 373812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 373912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 1 src, 1 dsr, int_ena, WXOR */ 374012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op); 374112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 374212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE); 374312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = PAGE_SIZE; 374412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 374512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 374612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rval = -EFAULT; 374712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 374812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto exit; 374912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 375012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 375112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 375212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Fill the test page with ones */ 375312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(page_address(pg), 0xFF, PAGE_SIZE); 375412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr = dma_map_page(chan->device->dev, pg, 0, 375512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 375612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 375712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup addresses */ 375812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0); 375912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0); 376012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addrs[0] = dma_addr; 376112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addrs[1] = 0; 376212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q); 376312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 376412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin async_tx_ack(&sw_desc->async_tx); 376512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.callback = ppc440spe_test_callback; 376612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.callback_param = NULL; 376712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 376812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin init_completion(&ppc440spe_r6_test_comp); 376912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 377012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_tx_submit(&sw_desc->async_tx); 377112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_issue_pending(&chan->common); 377212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 377312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin wait_for_completion(&ppc440spe_r6_test_comp); 377412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 377512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Now check if the test page is zeroed */ 377612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin a = page_address(pg); 377712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) { 377812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* page is zero - RAID-6 enabled */ 377912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rval = 0; 378012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 378112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RAID-6 was not enabled */ 378212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rval = -EINVAL; 378312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 378412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinexit: 378512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(pg); 378612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return rval; 378712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 378812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 378912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev) 379012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 379112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (adev->id) { 379212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 379312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 379412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_MEMCPY, adev->common.cap_mask); 379512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask); 379612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_PQ, adev->common.cap_mask); 379712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask); 379812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask); 379912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 380012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 380112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_XOR, adev->common.cap_mask); 380212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_PQ, adev->common.cap_mask); 380312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask); 380412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.cap_mask = adev->common.cap_mask; 380512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 380612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 380712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 380812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set base routines */ 380912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_alloc_chan_resources = 381012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_alloc_chan_resources; 381112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_free_chan_resources = 381212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_free_chan_resources; 38130793448187643b50af89d36b08470baf45a3cab4Linus Walleij adev->common.device_tx_status = ppc440spe_adma_tx_status; 381412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_issue_pending = ppc440spe_adma_issue_pending; 381512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 381612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set prep routines based on capability */ 381712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) { 381812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_memcpy = 381912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_memcpy; 382012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 382112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) { 382212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_xor = XOR_MAX_OPS; 382312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_xor = 382412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_xor; 382512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 382612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) { 382712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (adev->id) { 382812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 382912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_set_maxpq(&adev->common, 383012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0); 383112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 383212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 383312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_set_maxpq(&adev->common, 383412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0); 383512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 383612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 383712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_pq = XOR_MAX_OPS * 3; 383812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 383912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 384012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_pq = 384112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_pq; 384212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 384312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) { 384412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (adev->id) { 384512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 384612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_pq = DMA0_FIFO_SIZE / 384712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sizeof(struct dma_cdb); 384812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 384912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 385012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_pq = DMA1_FIFO_SIZE / 385112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sizeof(struct dma_cdb); 385212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 385312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 385412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_pq_val = 385512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_pqzero_sum; 385612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 385712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) { 385812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (adev->id) { 385912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 386012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_xor = DMA0_FIFO_SIZE / 386112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sizeof(struct dma_cdb); 386212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 386312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 386412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_xor = DMA1_FIFO_SIZE / 386512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sizeof(struct dma_cdb); 386612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 386712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 386812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_xor_val = 386912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_xor_zero_sum; 387012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 387112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) { 387212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_interrupt = 387312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_interrupt; 387412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 387512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: " 38766aa2731ce2c7bd1305b553b5fc14ae4856d36569Dan Williams "( %s%s%s%s%s%s)\n", 387712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_name(adev->dev), 387812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "", 387912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "", 388012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "", 388112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "", 388212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "", 388312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : ""); 388412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 388512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 388612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev, 388712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 388812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int *initcode) 388912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 38902dc11581376829303b98eadb2de253bee065a56aGrant Likely struct platform_device *ofdev; 389112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct device_node *np; 389212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int ret; 389312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 38942dc11581376829303b98eadb2de253bee065a56aGrant Likely ofdev = container_of(adev->dev, struct platform_device, dev); 38953e6b02d9f5a9715f7d4ff9e0978e5f9cef53d31fDan Williams np = ofdev->dev.of_node; 389612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id != PPC440SPE_XOR_ID) { 389712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->err_irq = irq_of_parse_and_map(np, 1); 389812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->err_irq == NO_IRQ) { 389912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_warn(adev->dev, "no err irq resource?\n"); 390012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *initcode = PPC_ADMA_INIT_IRQ2; 390112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->err_irq = -ENXIO; 390212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 390312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin atomic_inc(&ppc440spe_adma_err_irq_ref); 390412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 390512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->err_irq = -ENXIO; 390612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 390712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 390812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->irq = irq_of_parse_and_map(np, 0); 390912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->irq == NO_IRQ) { 391012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(adev->dev, "no irq resource\n"); 391112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *initcode = PPC_ADMA_INIT_IRQ1; 391212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENXIO; 391312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_irq_map; 391412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 391512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(adev->dev, "irq %d, err irq %d\n", 391612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->irq, adev->err_irq); 391712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 391812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = request_irq(adev->irq, ppc440spe_adma_eot_handler, 391912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 0, dev_driver_string(adev->dev), chan); 392012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) { 392112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(adev->dev, "can't request irq %d\n", 392212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->irq); 392312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *initcode = PPC_ADMA_INIT_IRQ1; 392412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -EIO; 392512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_req1; 392612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 392712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 392812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* only DMA engines have a separate error IRQ 392912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * so it's Ok if err_irq < 0 in XOR engine case. 393012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 393112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->err_irq > 0) { 393212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* both DMA engines share common error IRQ */ 393312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = request_irq(adev->err_irq, 393412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_err_handler, 393512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin IRQF_SHARED, 393612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_driver_string(adev->dev), 393712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan); 393812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) { 393912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(adev->dev, "can't request irq %d\n", 394012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->err_irq); 394112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *initcode = PPC_ADMA_INIT_IRQ2; 394212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -EIO; 394312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_req2; 394412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 394512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 394612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 394712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) { 394812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* enable XOR engine interrupts */ 394912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT | 395012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT, 395112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &adev->xor_reg->ier); 395212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 395312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 mask, enable; 395412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 395512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe"); 395612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!np) { 395712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't find I2O device tree node\n", 395812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 395912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 396012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_req2; 396112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 396212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->i2o_reg = of_iomap(np, 0); 396312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!adev->i2o_reg) { 396412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to map I2O registers\n", __func__); 396512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 396612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -EINVAL; 396712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_req2; 396812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 396912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 397012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Unmask 'CS FIFO Attention' interrupts and 397112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * enable generating interrupts on errors 397212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 397312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin enable = (adev->id == PPC440SPE_DMA0_ID) ? 397412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) : 397512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM); 397612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mask = ioread32(&adev->i2o_reg->iopim) & enable; 397712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(mask, &adev->i2o_reg->iopim); 397812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 397912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 398012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 398112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_req2: 398212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin free_irq(adev->irq, chan); 398312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_req1: 398412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin irq_dispose_mapping(adev->irq); 398512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_irq_map: 398612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->err_irq > 0) { 398712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) 398812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin irq_dispose_mapping(adev->err_irq); 398912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 399012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 399112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 399212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 399312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev, 399412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 399512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 399612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 mask, disable; 399712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 399812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) { 399912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* disable XOR engine interrupts */ 400012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mask = ioread32be(&adev->xor_reg->ier); 400112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT | 400212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT); 400312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(mask, &adev->xor_reg->ier); 400412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 400512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* disable DMAx engine interrupts */ 400612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin disable = (adev->id == PPC440SPE_DMA0_ID) ? 400712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) : 400812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM); 400912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mask = ioread32(&adev->i2o_reg->iopim) | disable; 401012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(mask, &adev->i2o_reg->iopim); 401112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 401212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin free_irq(adev->irq, chan); 401312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin irq_dispose_mapping(adev->irq); 401412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->err_irq > 0) { 401512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin free_irq(adev->err_irq, chan); 401612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) { 401712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin irq_dispose_mapping(adev->err_irq); 401812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->i2o_reg); 401912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 402012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 402112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 402212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 402312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 402412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_probe - probe the asynch device 402512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 4026463a1f8b3ceebe990ca9a8c7cc2e51ee42cb48ebBill Pembertonstatic int ppc440spe_adma_probe(struct platform_device *ofdev) 402712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 402805c02542c20aa00dc9a66f4bfb1a89d1131457f2Anatolij Gustschin struct device_node *np = ofdev->dev.of_node; 402912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct resource res; 403012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_device *adev; 403112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 403212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc_dma_chan_ref *ref, *_ref; 403312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int ret = 0, initcode = PPC_ADMA_INIT_OK; 403412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const u32 *idx; 403512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int len; 403612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin void *regs; 403712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 id, pool_size; 403812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 403912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (of_device_is_compatible(np, "amcc,xor-accelerator")) { 404012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin id = PPC440SPE_XOR_ID; 404112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* As far as the XOR engine is concerned, it does not 404212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * use FIFOs but uses linked list. So there is no dependency 404312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * between pool size to allocate and the engine configuration. 404412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 404512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pool_size = PAGE_SIZE << 1; 404612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 404712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* it is DMA0 or DMA1 */ 404812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idx = of_get_property(np, "cell-index", &len); 404912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!idx || (len != sizeof(u32))) { 405012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "Device node %s has missing " 405112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "or invalid cell-index property\n", 405212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np->full_name); 405312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 405412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 405512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin id = *idx; 405612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA0,1 engines use FIFO to maintain CDBs, so we 405712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * should allocate the pool accordingly to size of this 405812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * FIFO. Thus, the pool size depends on the FIFO depth: 405912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * how much CDBs pointers the FIFO may contain then so 406012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * much CDBs we should provide in the pool. 406112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * That is 406212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * CDB size = 32B; 406312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * CDBs number = (DMA0_FIFO_SIZE >> 3); 406412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Pool size = CDBs number * CDB size = 406512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2. 406612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 406712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pool_size = (id == PPC440SPE_DMA0_ID) ? 406812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA0_FIFO_SIZE : DMA1_FIFO_SIZE; 406912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pool_size <<= 2; 407012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 407112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 407212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (of_address_to_resource(np, 0, &res)) { 407312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to get memory resource\n"); 407412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_MEMRES; 407512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 407612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out; 407712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 407812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 407912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!request_mem_region(res.start, resource_size(&res), 408012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_driver_string(&ofdev->dev))) { 4081a584bff5efae8c1d026e3a930e3d13a90264fafcJoe Perches dev_err(&ofdev->dev, "failed to request memory region %pR\n", 4082a584bff5efae8c1d026e3a930e3d13a90264fafcJoe Perches &res); 408312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_MEMREG; 408412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -EBUSY; 408512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out; 408612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 408712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 408812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* create a device */ 408912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev = kzalloc(sizeof(*adev), GFP_KERNEL); 409012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!adev) { 409112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to allocate device\n"); 409212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_ALLOC; 409312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 409412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_adev_alloc; 409512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 409612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 409712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->id = id; 409812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->pool_size = pool_size; 409912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* allocate coherent memory for hardware descriptors */ 410012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev, 410112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->pool_size, &adev->dma_desc_pool, 410212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin GFP_KERNEL); 410312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->dma_desc_pool_virt == NULL) { 410412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent " 410512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "memory for hardware descriptors\n", 410612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->pool_size); 410712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_COHERENT; 410812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 410912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_dma_alloc; 411012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 4111d73111c6d43f08d697a3d986fe3bd8ca796a2096Masanari Iida dev_dbg(&ofdev->dev, "allocated descriptor pool virt 0x%p phys 0x%llx\n", 411212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool); 411312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 411412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin regs = ioremap(res.start, resource_size(&res)); 411512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!regs) { 411612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to ioremap regs!\n"); 4117f3b77727e82722fafb1940b3fc15bfdddd9aab4aJulia Lawall ret = -ENOMEM; 411812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_regs_alloc; 411912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 412012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 412112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) { 412212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->xor_reg = regs; 412312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Reset XOR */ 412412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr); 412512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr); 412612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 412712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ? 412812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA0_FIFO_SIZE : DMA1_FIFO_SIZE; 412912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_reg = regs; 413012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMAx_FIFO_SIZE is defined in bytes, 413112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * <fsiz> - is defined in number of CDB pointers (8byte). 413212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * DMA FIFO Length = CSlength + CPlength, where 413312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * CSlength = CPlength = (fsiz + 1) * 8. 413412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 413512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2), 413612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &adev->dma_reg->fsiz); 413712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Configure DMA engine */ 413812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN, 413912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &adev->dma_reg->cfg); 414012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Clear Status */ 414112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(~0, &adev->dma_reg->dsts); 414212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 414312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 414412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dev = &ofdev->dev; 414512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.dev = &ofdev->dev; 414612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&adev->common.channels); 4147dd3daca162f7411448dd80a8872a002c43cfd8e5Jingoo Han platform_set_drvdata(ofdev, adev); 414812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 414912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* create a channel */ 415012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = kzalloc(sizeof(*chan), GFP_KERNEL); 415112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!chan) { 415212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "can't allocate channel structure\n"); 415312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_CHANNEL; 415412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 415512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_chan_alloc; 415612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 415712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 415812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_init(&chan->lock); 415912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&chan->chain); 416012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&chan->all_slots); 416112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device = adev; 416212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->common.device = &adev->common; 41638ac695463f37af902e953d575d3f782e32e170daRussell King - ARM Linux dma_cookie_init(&chan->common); 416412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_add_tail(&chan->common.device_node, &adev->common.channels); 416512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_init(&chan->irq_tasklet, ppc440spe_adma_tasklet, 416612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (unsigned long)chan); 416712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 416812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* allocate and map helper pages for async validation or 416912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * async_mult/async_sum_product operations on DMA0/1. 417012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 417112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id != PPC440SPE_XOR_ID) { 417212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->pdest_page = alloc_page(GFP_KERNEL); 417312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->qdest_page = alloc_page(GFP_KERNEL); 417412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!chan->pdest_page || 417512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin !chan->qdest_page) { 417612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan->pdest_page) 417712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(chan->pdest_page); 417812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan->qdest_page) 417912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(chan->qdest_page); 418012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 418112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_page_alloc; 418212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 418312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0, 418412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 418512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0, 418612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 418712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 418812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 418912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ref = kmalloc(sizeof(*ref), GFP_KERNEL); 419012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ref) { 419112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ref->chan = &chan->common; 419212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&ref->node); 419312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_add_tail(&ref->node, &ppc440spe_adma_chan_list); 419412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 419512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to allocate channel reference!\n"); 419612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 419712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_ref_alloc; 419812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 419912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 420012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode); 420112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) 420212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_irq; 420312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 420412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_init_capabilities(adev); 420512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 420612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = dma_async_device_register(&adev->common); 420712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) { 420812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_REGISTER; 420912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to register dma device\n"); 421012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_dev_reg; 421112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 421212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 421312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out; 421412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 421512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_dev_reg: 421612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_release_irqs(adev, chan); 421712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_irq: 421812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) { 421912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan == to_ppc440spe_adma_chan(ref->chan)) { 422012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&ref->node); 422112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ref); 422212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 422312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 422412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_ref_alloc: 422512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id != PPC440SPE_XOR_ID) { 422612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(&ofdev->dev, chan->pdest, 422712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 422812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(&ofdev->dev, chan->qdest, 422912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 423012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(chan->pdest_page); 423112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(chan->qdest_page); 423212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 423312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_page_alloc: 423412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(chan); 423512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_chan_alloc: 423612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) 423712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->xor_reg); 423812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 423912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->dma_reg); 424012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_regs_alloc: 424112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_free_coherent(adev->dev, adev->pool_size, 424212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool_virt, 424312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool); 424412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_dma_alloc: 424512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(adev); 424612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_adev_alloc: 424712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin release_mem_region(res.start, resource_size(&res)); 424812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout: 424912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (id < PPC440SPE_ADMA_ENGINES_NUM) 425012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_devices[id] = initcode; 425112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 425212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 425312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 425412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 425512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 425612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_remove - remove the asynch device 425712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 42584bf27b8b333bcd291664fd0f7d129099d474a23bGreg Kroah-Hartmanstatic int ppc440spe_adma_remove(struct platform_device *ofdev) 425912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 4260dd3daca162f7411448dd80a8872a002c43cfd8e5Jingoo Han struct ppc440spe_adma_device *adev = platform_get_drvdata(ofdev); 426105c02542c20aa00dc9a66f4bfb1a89d1131457f2Anatolij Gustschin struct device_node *np = ofdev->dev.of_node; 426212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct resource res; 426312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, *_chan; 426412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc_dma_chan_ref *ref, *_ref; 426512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 426612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 426712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id < PPC440SPE_ADMA_ENGINES_NUM) 426812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_devices[adev->id] = -1; 426912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 427012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_async_device_unregister(&adev->common); 427112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 427212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(chan, _chan, &adev->common.channels, 427312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin device_node) { 427412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 427512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_release_irqs(adev, ppc440spe_chan); 427612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_kill(&ppc440spe_chan->irq_tasklet); 427712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id != PPC440SPE_XOR_ID) { 427812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest, 427912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 428012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest, 428112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 428212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(ppc440spe_chan->pdest_page); 428312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(ppc440spe_chan->qdest_page); 428412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 428512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, 428612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin node) { 428712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_chan == 428812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin to_ppc440spe_adma_chan(ref->chan)) { 428912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&ref->node); 429012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ref); 429112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 429212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 429312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&chan->device_node); 429412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ppc440spe_chan); 429512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 429612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 429712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_free_coherent(adev->dev, adev->pool_size, 429812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool_virt, adev->dma_desc_pool); 429912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) 430012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->xor_reg); 430112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 430212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->dma_reg); 430312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_address_to_resource(np, 0, &res); 430412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin release_mem_region(res.start, resource_size(&res)); 430512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(adev); 430612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 430712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 430812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 430912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* 431012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * /sys driver interface to enable h/w RAID-6 capabilities 431112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/ 431212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * directory are "devices", "enable" and "poly". 431312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * "devices" shows available engines. 431412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * "enable" is used to enable RAID-6 capabilities or to check 431512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * whether these has been activated. 431612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * "poly" allows setting/checking used polynomial (for PPC440SPe only). 431712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 431812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 431912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t show_ppc440spe_devices(struct device_driver *dev, char *buf) 432012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 432112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ssize_t size = 0; 432212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 432312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 432412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) { 432512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_adma_devices[i] == -1) 432612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 432712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size += snprintf(buf + size, PAGE_SIZE - size, 432812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "PPC440SP(E)-ADMA.%d: %s\n", i, 432912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc_adma_errors[ppc440spe_adma_devices[i]]); 433012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 433112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return size; 433212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 433312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 433412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t show_ppc440spe_r6enable(struct device_driver *dev, char *buf) 433512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 433612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return snprintf(buf, PAGE_SIZE, 433712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "PPC440SP(e) RAID-6 capabilities are %sABLED.\n", 433812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_r6_enabled ? "EN" : "DIS"); 433912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 434012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 434112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t store_ppc440spe_r6enable(struct device_driver *dev, 434212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const char *buf, size_t count) 434312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 434412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long val; 434512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 434612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!count || count > 11) 434712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 434812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 434912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!ppc440spe_r6_tchan) 435012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EFAULT; 435112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 435212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Write a key */ 435312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sscanf(buf, "%lx", &val); 435412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val); 435512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin isync(); 435612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 435712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Verify whether it really works now */ 435812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) { 435912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_info("PPC440SP(e) RAID-6 has been activated " 436012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "successfully\n"); 436112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_r6_enabled = 1; 436212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 436312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_info("PPC440SP(e) RAID-6 hasn't been activated!" 436412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " Error key ?\n"); 436512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_r6_enabled = 0; 436612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 436712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return count; 436812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 436912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 437012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t show_ppc440spe_r6poly(struct device_driver *dev, char *buf) 437112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 437212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ssize_t size = 0; 437312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 reg; 437412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 437512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#ifdef CONFIG_440SP 437612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 440SP has fixed polynomial */ 437712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg = 0x4d; 437812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#else 437912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL); 438012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg >>= MQ0_CFBHL_POLY; 438112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg &= 0xFF; 438212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#endif 438312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 438412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver " 438512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "uses 0x1%02x polynomial.\n", reg); 438612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return size; 438712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 438812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 438912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t store_ppc440spe_r6poly(struct device_driver *dev, 439012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const char *buf, size_t count) 439112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 439212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long reg, val; 439312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 439412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#ifdef CONFIG_440SP 439512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 440SP uses default 0x14D polynomial only */ 439612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 439712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#endif 439812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 439912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!count || count > 6) 440012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 440112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 440212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* e.g., 0x14D or 0x11D */ 440312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sscanf(buf, "%lx", &val); 440412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 440512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (val & ~0x1FF) 440612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 440712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 440812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin val &= 0xFF; 440912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL); 441012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg &= ~(0xFF << MQ0_CFBHL_POLY); 441112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg |= val << MQ0_CFBHL_POLY; 441212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg); 441312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 441412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return count; 441512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 441612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 441712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic DRIVER_ATTR(devices, S_IRUGO, show_ppc440spe_devices, NULL); 441812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic DRIVER_ATTR(enable, S_IRUGO | S_IWUSR, show_ppc440spe_r6enable, 441912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin store_ppc440spe_r6enable); 442012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic DRIVER_ATTR(poly, S_IRUGO | S_IWUSR, show_ppc440spe_r6poly, 442112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin store_ppc440spe_r6poly); 442212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 442312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* 442412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Common initialisation for RAID engines; allocate memory for 442512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * DMAx FIFOs, perform configuration common for all DMA engines. 442612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Further DMA engine specific configuration is done at probe time. 442712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 442812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_configure_raid_devices(void) 442912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 443012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct device_node *np; 443112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct resource i2o_res; 443212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct i2o_regs __iomem *i2o_reg; 443312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_host_t i2o_dcr_host; 443412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int dcr_base, dcr_len; 443512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i, ret; 443612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 443712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe"); 443812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!np) { 443912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't find I2O device tree node\n", 444012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 444112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENODEV; 444212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 444312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 444412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (of_address_to_resource(np, 0, &i2o_res)) { 444512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 444612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 444712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 444812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 444912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i2o_reg = of_iomap(np, 0); 445012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!i2o_reg) { 445112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to map I2O registers\n", __func__); 445212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 445312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 445412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 445512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 445612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Get I2O DCRs base */ 445712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_base = dcr_resource_start(np, 0); 445812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_len = dcr_resource_len(np, 0); 445912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!dcr_base && !dcr_len) { 446012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't get DCR registers base/len!\n", 446112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np->full_name); 446212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 446312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(i2o_reg); 446412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENODEV; 446512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 446612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 446712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i2o_dcr_host = dcr_map(np, dcr_base, dcr_len); 446812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!DCR_MAP_OK(i2o_dcr_host)) { 446912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to map DCRs!\n", np->full_name); 447012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 447112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(i2o_reg); 447212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENODEV; 447312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 447412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 447512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 447612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share 447712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the base address of FIFO memory space. 447812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Actually we need twice more physical memory than programmed in the 447912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * <fsiz> register (because there are two FIFOs for each DMA: CP and CS) 448012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 448112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1, 448212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin GFP_KERNEL); 448312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!ppc440spe_dma_fifo_buf) { 448412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__); 448512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(i2o_reg); 448612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_unmap(i2o_dcr_host, dcr_len); 448712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENOMEM; 448812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 448912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 449012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 449112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Configure h/w 449212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 449312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Reset I2O/DMA */ 449412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA); 449512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mtdcri(SDR0, DCRN_SDR0_SRST, 0); 449612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 449712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup the base address of mmaped registers */ 449812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32)); 449912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) | 450012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin I2O_REG_ENABLE); 450112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_unmap(i2o_dcr_host, dcr_len); 450212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 450312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup FIFO memory space base address */ 450412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(0, &i2o_reg->ifbah); 450512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal); 450612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 450712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set zero FIFO size for I2O, so the whole 450812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma_fifo_buf is used by DMAs. 450912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * DMAx_FIFOs will be configured while probe. 451012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 451112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(0, &i2o_reg->ifsiz); 451212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(i2o_reg); 451312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 451412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* To prepare WXOR/RXOR functionality we need access to 451512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Memory Queue Module DCRs (finally it will be enabled 451612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * via /sys interface of the ppc440spe ADMA driver). 451712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 451812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe"); 451912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!np) { 452012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't find MQ device tree node\n", 452112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 452212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 452312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_free; 452412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 452512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 452612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Get MQ DCRs base */ 452712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_base = dcr_resource_start(np, 0); 452812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_len = dcr_resource_len(np, 0); 452912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!dcr_base && !dcr_len) { 453012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't get DCR registers base/len!\n", 453112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np->full_name); 453212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 453312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_mq; 453412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 453512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 453612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len); 453712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) { 453812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to map DCRs!\n", np->full_name); 453912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 454012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_mq; 454112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 454212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 454312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_mq_dcr_len = dcr_len; 454412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 454512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set HB alias */ 454612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB); 454712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 454812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set: 454912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - LL transaction passing limit to 1; 455012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - Memory controller cycle limit to 1; 455112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - Galois Polynomial to 0x14d (default) 455212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 455312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, 455412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) | 455512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY)); 455612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 455712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin atomic_set(&ppc440spe_adma_err_irq_ref, 0); 455812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) 455912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_devices[i] = -1; 456012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 456112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 456212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 456312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_mq: 456412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 456512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_free: 456612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ppc440spe_dma_fifo_buf); 456712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 456812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 456912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 45704bf27b8b333bcd291664fd0f7d129099d474a23bGreg Kroah-Hartmanstatic const struct of_device_id ppc440spe_adma_of_match[] = { 457112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin { .compatible = "ibm,dma-440spe", }, 457212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin { .compatible = "amcc,xor-accelerator", }, 457312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin {}, 457412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 457512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinMODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match); 457612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 4577000061245a6797d542854106463b6b20fbdcb12eGrant Likelystatic struct platform_driver ppc440spe_adma_driver = { 457812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin .probe = ppc440spe_adma_probe, 4579a7d6e3ec28bba30d1409d70de1958edc6d9109d7Bill Pemberton .remove = ppc440spe_adma_remove, 458012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin .driver = { 458112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin .name = "PPC440SP(E)-ADMA", 458212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin .owner = THIS_MODULE, 45834018294b53d1dae026880e45f174c1cc63b5d435Grant Likely .of_match_table = ppc440spe_adma_of_match, 458412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin }, 458512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 458612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 458712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic __init int ppc440spe_adma_init(void) 458812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 458912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int ret; 459012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 459112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = ppc440spe_configure_raid_devices(); 459212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) 459312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 459412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 4595000061245a6797d542854106463b6b20fbdcb12eGrant Likely ret = platform_driver_register(&ppc440spe_adma_driver); 459612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) { 459712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to register platform driver\n", 459812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 459912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_reg; 460012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 460112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 460212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Initialization status */ 460312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = driver_create_file(&ppc440spe_adma_driver.driver, 460412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_devices); 460512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) 460612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_dev; 460712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 460812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RAID-6 h/w enable entry */ 460912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = driver_create_file(&ppc440spe_adma_driver.driver, 461012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_enable); 461112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) 461212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_en; 461312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 461412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* GF polynomial to use */ 461512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = driver_create_file(&ppc440spe_adma_driver.driver, 461612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_poly); 461712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!ret) 461812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 461912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 462012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 462112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_enable); 462212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_en: 462312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 462412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_devices); 462512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_dev: 462612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* User will not be able to enable h/w RAID-6 */ 462712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to create RAID-6 driver interface\n", 462812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 4629000061245a6797d542854106463b6b20fbdcb12eGrant Likely platform_driver_unregister(&ppc440spe_adma_driver); 463012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_reg: 463112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len); 463212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ppc440spe_dma_fifo_buf); 463312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 463412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 463512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 463612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void __exit ppc440spe_adma_exit(void) 463712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 463812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 463912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_poly); 464012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 464112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_enable); 464212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 464312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_devices); 4644000061245a6797d542854106463b6b20fbdcb12eGrant Likely platform_driver_unregister(&ppc440spe_adma_driver); 464512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len); 464612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ppc440spe_dma_fifo_buf); 464712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 464812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 464912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinarch_initcall(ppc440spe_adma_init); 465012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinmodule_exit(ppc440spe_adma_exit); 465112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 465212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinMODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>"); 465312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinMODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver"); 465412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinMODULE_LICENSE("GPL"); 4655