adma.c revision c11eede69b6ad0ac44ebc1e021a8d2699c5f1f8f
112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* 212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Copyright (C) 2006-2009 DENX Software Engineering. 312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Author: Yuri Tikhonov <yur@emcraft.com> 512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Further porting to arch/powerpc by 712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Anatolij Gustschin <agust@denx.de> 812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * This program is free software; you can redistribute it and/or modify it 1012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * under the terms of the GNU General Public License as published by the Free 1112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Software Foundation; either version 2 of the License, or (at your option) 1212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * any later version. 1312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 1412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * This program is distributed in the hope that it will be useful, but WITHOUT 1512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 1612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 1712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * more details. 1812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 1912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * You should have received a copy of the GNU General Public License along with 2012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * this program; if not, write to the Free Software Foundation, Inc., 59 2112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Temple Place - Suite 330, Boston, MA 02111-1307, USA. 2212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 2312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * The full GNU General Public License is included in this distribution in the 2412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * file called COPYING. 2512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 2612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 2712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* 2812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * This driver supports the asynchrounous DMA copy and RAID engines available 2912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * on the AMCC PPC440SPe Processors. 3012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) 3112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ADMA driver written by D.Williams. 3212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 3312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 3412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/init.h> 3512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/module.h> 3612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/async_tx.h> 3712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/delay.h> 3812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/dma-mapping.h> 3912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/spinlock.h> 4012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/interrupt.h> 415a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 4212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/uaccess.h> 4312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/proc_fs.h> 4412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/of.h> 45c11eede69b6ad0ac44ebc1e021a8d2699c5f1f8fRob Herring#include <linux/of_address.h> 46c11eede69b6ad0ac44ebc1e021a8d2699c5f1f8fRob Herring#include <linux/of_irq.h> 4712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <linux/of_platform.h> 4812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <asm/dcr.h> 4912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include <asm/dcr-regs.h> 5012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#include "adma.h" 51d2ebfb335b0426deb1a4fb14e4e926d81ecd8235Russell King - ARM Linux#include "../dmaengine.h" 5212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 5312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinenum ppc_adma_init_code { 5412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_OK = 0, 5512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_MEMRES, 5612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_MEMREG, 5712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_ALLOC, 5812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_COHERENT, 5912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_CHANNEL, 6012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_IRQ1, 6112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_IRQ2, 6212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PPC_ADMA_INIT_REGISTER 6312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 6412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 6512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic char *ppc_adma_errors[] = { 6612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_OK] = "ok", 6712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource", 6812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_MEMREG] = "failed to request memory region", 6912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev " 7012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "structure", 7112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for " 7212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "hardware descriptors", 7312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel", 7412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_IRQ1] = "failed to request first irq", 7512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_IRQ2] = "failed to request second irq", 7612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device", 7712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 7812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 7912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic enum ppc_adma_init_code 8012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM]; 8112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 8212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstruct ppc_dma_chan_ref { 8312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan; 8412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct list_head node; 8512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 8612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 8712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* The list of channels exported by ppc440spe ADMA */ 8812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstruct list_head 8912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list); 9012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 9112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* This flag is set when want to refetch the xor chain in the interrupt 9212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * handler 9312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 9412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 do_xor_refetch; 9512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 9612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* Pointer to DMA0, DMA1 CP/CS FIFO */ 9712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void *ppc440spe_dma_fifo_buf; 9812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 9912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* Pointers to last submitted to DMA0, DMA1 CDBs */ 10012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *chan_last_sub[3]; 10112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *chan_first_cdb[3]; 10212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 10312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* Pointer to last linked and submitted xor CB */ 10412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *xor_last_linked; 10512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *xor_last_submit; 10612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 10712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* This array is used in data-check operations for storing a pattern */ 10812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic char ppc440spe_qword[16]; 10912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 11012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic atomic_t ppc440spe_adma_err_irq_ref; 11112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic dcr_host_t ppc440spe_mq_dcr_host; 11212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic unsigned int ppc440spe_mq_dcr_len; 11312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 11412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* Since RXOR operations use the common register (MQ0_CF2H) for setting-up 11512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the block size in transactions, then we do not allow to activate more than 11612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * only one RXOR transactions simultaneously. So use this var to store 11712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is 11812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * set) or not (PPC440SPE_RXOR_RUN is clear). 11912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 12012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic unsigned long ppc440spe_rxor_state; 12112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 12212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* These are used in enable & check routines 12312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 12412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 ppc440spe_r6_enabled; 12512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_chan *ppc440spe_r6_tchan; 12612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct completion ppc440spe_r6_test_comp; 12712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 12812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_dma2rxor_prep_src( 12912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 13012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_rxor *cursor, int index, 13112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, u32 addr); 13212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_set_src( 13312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 13412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index, dma_addr_t addr); 13512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_set_mult( 13612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 13712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index, u8 mult); 13812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 13912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#ifdef ADMA_LL_DEBUG 14012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define ADMA_LL_DBG(x) ({ if (1) x; 0; }) 14112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#else 14212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define ADMA_LL_DBG(x) ({ if (0) x; 0; }) 14312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#endif 14412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 14512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void print_cb(struct ppc440spe_adma_chan *chan, void *block) 14612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 14712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *cdb; 14812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *cb; 14912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 15012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 15112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 15212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 0: 15312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 1: 15412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb = block; 15512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 15612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("CDB at %p [%d]:\n" 15712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n" 15812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t sg1u 0x%08x sg1l 0x%08x\n" 15912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t sg2u 0x%08x sg2l 0x%08x\n" 16012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t sg3u 0x%08x sg3l 0x%08x\n", 16112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb, chan->device->id, 16212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt), 16312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l), 16412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l), 16512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l) 16612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ); 16712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 16812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 2: 16912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb = block; 17012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 17112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("CB at %p [%d]:\n" 17212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n" 17312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t cbtah 0x%08x cbtal 0x%08x\n" 17412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "\t cblah 0x%08x cblal 0x%08x\n", 17512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb, chan->device->id, 17612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb->cbc, cb->cbbc, cb->cbs, 17712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb->cbtah, cb->cbtal, 17812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cb->cblah, cb->cblal); 17912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < 16; i++) { 18012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i && !cb->ops[i].h && !cb->ops[i].l) 18112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 18212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n", 18312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i, cb->ops[i].h, cb->ops[i].l); 18412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 18512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 18612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 18712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 18812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 18912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void print_cb_list(struct ppc440spe_adma_chan *chan, 19012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter) 19112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 19212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (; iter; iter = iter->hw_next) 19312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin print_cb(chan, iter->hw_desc); 19412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 19512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 19612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src, 19712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt) 19812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 19912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 20012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 20112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\n%s(%d):\nsrc: ", __func__, id); 20212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 20312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx ", src[i]); 20412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("dst:\n\t0x%016llx\n", dst); 20512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 20612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 20712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src, 20812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt) 20912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 21012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 21112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 21212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\n%s(%d):\nsrc: ", __func__, id); 21312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 21412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx ", src[i]); 21512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("dst: "); 21612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < 2; i++) 21712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx ", dst[i]); 21812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 21912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 22012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src, 22112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt, 22212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf) 22312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 22412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 22512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 22612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id); 22712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (scf) { 22812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 22912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]); 23012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 23112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 23212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx(no) ", src[i]); 23312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 23412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 23512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("dst: "); 23612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < 2; i++) 23712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\t0x%016llx ", src[src_cnt + i]); 23812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 23912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 24012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/****************************************************************************** 24112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Command (Descriptor) Blocks low-level routines 24212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ******************************************************************************/ 24312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 24412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT 24512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * pseudo operation 24612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 24712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc, 24812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 24912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 25012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *p; 25112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 25212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 25312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 25412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin p = desc->hw_desc; 25512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 25612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* NOP with Command Block Complete Enable */ 25712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin p->cbc = XOR_CBCR_CBCE_BIT; 25812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 25912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 26012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 26112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 26212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* NOP with interrupt */ 26312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &desc->flags); 26412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 26512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 26612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id, 26712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 26812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 26912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 27012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 27112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 27212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 27312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR 27412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * pseudo operation 27512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 27612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc) 27712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 27812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 27912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 28012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = 0; 28112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = 1; 28212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 28312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 28412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 28512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation 28612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 28712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc, 28812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, unsigned long flags) 28912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 29012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *hw_desc = desc->hw_desc; 29112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 29212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 29312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 29412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = src_cnt; 29512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = 1; 29612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 29712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt; 29812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 29912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Enable interrupt on completion */ 30012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc |= XOR_CBCR_CBCE_BIT; 30112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 30212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 30312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 30412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ 30512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * operation in DMA2 controller 30612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 30712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc, 30812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int dst_cnt, int src_cnt, unsigned long flags) 30912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 31012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *hw_desc = desc->hw_desc; 31112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 31212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 31312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 31412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = src_cnt; 31512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = dst_cnt; 31612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags)); 31712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->descs_per_op = 0; 31812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 31912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc = XOR_CBCR_TGT_BIT; 32012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 32112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Enable interrupt on completion */ 32212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc |= XOR_CBCR_CBCE_BIT; 32312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 32412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 32512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define DMA_CTRL_FLAGS_LAST DMA_PREP_FENCE 32612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define DMA_PREP_ZERO_P (DMA_CTRL_FLAGS_LAST << 1) 32712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#define DMA_PREP_ZERO_Q (DMA_PREP_ZERO_P << 1) 32812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 32912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 33012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation 33112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * with DMA0/1 33212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 33312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc, 33412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int dst_cnt, int src_cnt, unsigned long flags, 33512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op) 33612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 33712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 33812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 33912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 dopc; 34012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 34112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Common initialization of a PQ descriptors chain */ 34212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bits(op, &desc->flags); 34312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = src_cnt; 34412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = dst_cnt; 34512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 34612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR MULTICAST if both P and Q are being computed 34712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * MV_SG1_SG2 if Q only 34812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 34912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ? 35012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2; 35112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 35212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &desc->group_list, chain_node) { 35312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 35412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 35512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 35612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(!list_is_last(&iter->chain_node, 35712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &desc->group_list))) { 35812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 35912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 36012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 36112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 36212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 36312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is the last descriptor. 36412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * this slot will be pasted from ADMA level 36512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * each time it wants to configure parameters 36612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * of the transaction (src, dst, ...) 36712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 36812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 36912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 37012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 37112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 37212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 37312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 37412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 37512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 37612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set OPS depending on WXOR/RXOR type of operation */ 37712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) { 37812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is a WXOR only chain: 37912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - first descriptors are for zeroing destinations 38012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * if PPC440SPE_ZERO_P/Q set; 38112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - descriptors remained are for GF-XOR operations. 38212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 38312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&desc->group_list, 38412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 38512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 38612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 38712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) { 38812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 38912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 39012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 39112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 39212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 39312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 39412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 39512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) { 39612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 39712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 39812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 39912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 40012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 40112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 40212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 40312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &desc->group_list, chain_node) { 40412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 40512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = dopc; 40612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 40712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 40812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is either RXOR-only or mixed RXOR/WXOR */ 40912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 41012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* The first 1 or 2 slots in chain are always RXOR, 41112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * if need to calculate P & Q, then there are two 41212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * RXOR slots; if only P or only Q, then there is one 41312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 41412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&desc->group_list, 41512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 41612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 41712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 41812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 41912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 42012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (desc->dst_cnt == DMA_DEST_MAX_NUM) { 42112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 42212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 42312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 42412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 42512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 42612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 42712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 42812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* The remaining descs (if any) are WXORs */ 42912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) { 43012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 43112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 43212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 43312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &desc->group_list, 43412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 43512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 43612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = dopc; 43712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 43812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 43912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 44012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 44112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 44212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 44312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor 44412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for PQ_ZERO_SUM operation 44512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 44612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_dma01pqzero_sum( 44712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 44812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int dst_cnt, int src_cnt) 44912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 45012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 45112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 45212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i = 0; 45312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST : 45412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_OPC_MV_SG1_SG2; 45512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 45612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Initialize starting from 2nd or 3rd descriptor dependent 45712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * on dst_cnt. First one or two slots are for cloning P 45812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * and/or Q to chan->pdest and/or chan->qdest as we have 45912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * to preserve original P/Q. 46012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 46112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&desc->group_list, 46212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 46312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 46412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 46512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 46612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dst_cnt > 1) { 46712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 46812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 46912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 47012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* initialize each source descriptor in chain */ 47112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &desc->group_list, chain_node) { 47212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 47312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 47412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->src_cnt = 0; 47512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->dst_cnt = 0; 47612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 47712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is a ZERO_SUM operation: 47812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - <src_cnt> descriptors starting from 2nd or 3rd 47912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor are for GF-XOR operations; 48012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - remaining <dst_cnt> descriptors are for checking the result 48112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 48212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i++ < src_cnt) 48312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* MV_SG1_SG2 if only Q is being verified 48412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * MULTICAST if both P and Q are being verified 48512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 48612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = dopc; 48712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 48812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA_CDB_OPC_DCHECK128 operation */ 48912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_DCHECK128; 49012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 49112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(!list_is_last(&iter->chain_node, 49212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &desc->group_list))) { 49312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 49412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 49512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 49612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 49712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 49812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is the last descriptor. 49912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * this slot will be pasted from ADMA level 50012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * each time it wants to configure parameters 50112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * of the transaction (src, dst, ...) 50212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 50312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 50412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* always enable interrupt generation since we get 50512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the status of pqzero from the handler 50612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 50712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 50812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 50912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 51012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = src_cnt; 51112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = dst_cnt; 51212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 51312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 51412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 51512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation 51612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 51712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc, 51812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long flags) 51912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 52012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc = desc->hw_desc; 52112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 52212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 52312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 52412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = 1; 52512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = 1; 52612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 52712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 52812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &desc->flags); 52912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 53012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &desc->flags); 53112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 53212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 53312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 53412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 53512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 53612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_init_memset - initialize the descriptor for MEMSET operation 53712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 53812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_init_memset(struct ppc440spe_adma_desc_slot *desc, 53912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int value, unsigned long flags) 54012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 54112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc = desc->hw_desc; 54212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 54312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 54412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->hw_next = NULL; 54512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt = 1; 54612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->dst_cnt = 1; 54712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 54812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 54912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &desc->flags); 55012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 55112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &desc->flags); 55212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 55312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->sg1u = hw_desc->sg1l = cpu_to_le32((u32)value); 55412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->sg3u = hw_desc->sg3l = cpu_to_le32((u32)value); 55512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_DFILL128; 55612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 55712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 55812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 55912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_src_addr - set source address into the descriptor 56012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 56112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc, 56212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 56312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_idx, dma_addr_t addrh, 56412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addrl) 56512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 56612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 56712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 56812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin phys_addr_t addr64, tmplow, tmphi; 56912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 57012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 57112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 57212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 57312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addrh) { 57412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr64 = addrl; 57512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmphi = (addr64 >> 32); 57612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmplow = (addr64 & 0xFFFFFFFF); 57712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 57812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmphi = addrh; 57912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmplow = addrl; 58012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 58112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 58212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow); 58312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi); 58412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 58512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 58612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 58712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->ops[src_idx].l = addrl; 58812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->ops[src_idx].h |= addrh; 58912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 59012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 59112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 59212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 59312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 59412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_src_mult - set source address mult into the descriptor 59512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 59612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc, 59712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, u32 mult_index, 59812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int sg_index, unsigned char mult_value) 59912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 60012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 60112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 60212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 *psgu; 60312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 60412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 60512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 60612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 60712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 60812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 60912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (sg_index) { 61012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* for RXOR operations set multiplier 61112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * into source cued address 61212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 61312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_SG_SRC: 61412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgu = &dma_hw_desc->sg1u; 61512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 61612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* for WXOR operations set multiplier 61712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * into destination cued address(es) 61812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 61912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_SG_DST1: 62012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgu = &dma_hw_desc->sg2u; 62112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 62212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_SG_DST2: 62312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgu = &dma_hw_desc->sg3u; 62412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 62512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 62612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 62712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 62812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 62912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *psgu |= cpu_to_le32(mult_value << mult_index); 63012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 63112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 63212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 63312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 63412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 63512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 63612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 63712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 63812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 63912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 64012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_dest_addr - set destination address into the descriptor 64112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 64212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc, 64312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 64412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addrh, dma_addr_t addrl, 64512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 dst_idx) 64612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 64712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 64812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 64912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin phys_addr_t addr64, tmphi, tmplow; 65012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 *psgu, *psgl; 65112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 65212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 65312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 65412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 65512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addrh) { 65612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr64 = addrl; 65712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmphi = (addr64 >> 32); 65812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmplow = (addr64 & 0xFFFFFFFF); 65912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 66012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmphi = addrh; 66112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tmplow = addrl; 66212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 66312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 66412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 66512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u; 66612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l; 66712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 66812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *psgl = cpu_to_le32((u32)tmplow); 66912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *psgu |= cpu_to_le32((u32)tmphi); 67012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 67112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 67212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 67312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbtal = addrl; 67412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbtah |= addrh; 67512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 67612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 67712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 67812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 67912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 68012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_byte_count - set number of data bytes involved 68112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * into the operation 68212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 68312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc, 68412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 68512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 byte_count) 68612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 68712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 68812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 68912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 69012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 69112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 69212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 69312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 69412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->cnt = cpu_to_le32(byte_count); 69512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 69612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 69712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 69812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbbc = byte_count; 69912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 70012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 70112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 70212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 70312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 70412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_rxor_block_size - set RXOR block size 70512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 70612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count) 70712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 70812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* assume that byte_count is aligned on the 512-boundary; 70912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * thus write it directly to the register (bits 23:31 are 71012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * reserved there). 71112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 71212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count); 71312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 71412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 71512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 71612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_dcheck - set CHECK pattern 71712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 71812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc, 71912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, u8 *qword) 72012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 72112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 72212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 72312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 72412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 72512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 72612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 72712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(qword[0], &dma_hw_desc->sg3l); 72812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(qword[4], &dma_hw_desc->sg3u); 72912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(qword[8], &dma_hw_desc->sg2l); 73012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(qword[12], &dma_hw_desc->sg2u); 73112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 73212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 73312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 73412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 73512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 73612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 73712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 73812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_xor_set_link - set link address in xor CB 73912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 74012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc, 74112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *next_desc) 74212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 74312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc = prev_desc->hw_desc; 74412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 74512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!next_desc || !(next_desc->phys))) { 74612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n", 74712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, next_desc, 74812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin next_desc ? next_desc->phys : 0); 74912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 75012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 75112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 75212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbs = 0; 75312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cblal = next_desc->phys; 75412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cblah = 0; 75512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT; 75612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 75712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 75812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 75912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_link - set the address of descriptor following this 76012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor in chain 76112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 76212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan, 76312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *prev_desc, 76412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *next_desc) 76512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 76612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long flags; 76712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *tail = next_desc; 76812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 76912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!prev_desc || !next_desc || 77012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (prev_desc->hw_next && prev_desc->hw_next != next_desc))) { 77112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* If previous next is overwritten something is wrong. 77212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * though we may refetch from append to initiate list 77312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * processing; in this case - it's ok. 77412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 77512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; " 77612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "prev->hw_next=0x%p\n", __func__, prev_desc, 77712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin next_desc, prev_desc ? prev_desc->hw_next : 0); 77812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 77912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 78012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 78112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin local_irq_save(flags); 78212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 78312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do s/w chaining both for DMA and XOR descriptors */ 78412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prev_desc->hw_next = next_desc; 78512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 78612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 78712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 78812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 78912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 79012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 79112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* bind descriptor to the chain */ 79212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (tail->hw_next) 79312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tail = tail->hw_next; 79412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_linked = tail; 79512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 79612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (prev_desc == xor_last_submit) 79712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do not link to the last submitted CB */ 79812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 79912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_xor_set_link(prev_desc, next_desc); 80012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 80112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 80212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 80312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin local_irq_restore(flags); 80412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 80512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 80612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 80712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_get_src_addr - extract the source address from the descriptor 80812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 80912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 ppc440spe_desc_get_src_addr(struct ppc440spe_adma_desc_slot *desc, 81012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, int src_idx) 81112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 81212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 81312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 81412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 81512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 81612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 81712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 81812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 81912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* May have 0, 1, 2, or 3 sources */ 82012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (dma_hw_desc->opc) { 82112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_NO_OP: 82212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DFILL128: 82312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 82412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DCHECK128: 82512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(src_idx)) { 82612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: try to get %d source for" 82712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " DCHECK128\n", __func__, src_idx); 82812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 82912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 83012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu(dma_hw_desc->sg1l); 83112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MULTICAST: 83212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MV_SG1_SG2: 83312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(src_idx > 2)) { 83412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: try to get %d source from" 83512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " DMA descr\n", __func__, src_idx); 83612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 83712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 83812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_idx) { 83912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (le32_to_cpu(dma_hw_desc->sg1u) & 84012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_WIN_MSK) { 84112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 region; 84212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 84312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_idx == 1) 84412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu( 84512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1l) + 84612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->unmap_len; 84712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 84812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin region = (le32_to_cpu( 84912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1u)) >> 85012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 85112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 85212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin region &= DMA_CUED_REGION_MSK; 85312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (region) { 85412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_RXOR123: 85512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu( 85612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1l) + 85712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (desc->unmap_len << 1); 85812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_RXOR124: 85912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu( 86012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1l) + 86112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (desc->unmap_len * 3); 86212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_RXOR125: 86312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu( 86412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc->sg1l) + 86512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (desc->unmap_len << 2); 86612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 86712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR 86812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "%s: try to" 86912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " get src3 for region %02x" 87012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "PPC440SPE_DESC_RXOR12?\n", 87112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, region); 87212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 87312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 87412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 87512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR 87612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "%s: try to get %d" 87712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " source for non-cued descr\n", 87812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, src_idx); 87912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 88012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 88112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 88212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu(dma_hw_desc->sg1l); 88312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 88412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: unknown OPC 0x%02x\n", 88512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dma_hw_desc->opc); 88612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 88712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 88812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu(dma_hw_desc->sg1l); 88912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 89012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* May have up to 16 sources */ 89112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 89212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return xor_hw_desc->ops[src_idx].l; 89312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 89412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 89512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 89612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 89712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 89812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_get_dest_addr - extract the destination address from the 89912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor 90012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 90112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 ppc440spe_desc_get_dest_addr(struct ppc440spe_adma_desc_slot *desc, 90212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, int idx) 90312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 90412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 90512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 90612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 90712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 90812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 90912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 91012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 91112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 91212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(!idx)) 91312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu(dma_hw_desc->sg2l); 91412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return le32_to_cpu(dma_hw_desc->sg3l); 91512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 91612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 91712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return xor_hw_desc->cbtal; 91812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 91912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 92012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 92112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 92212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 92312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_get_src_num - extract the number of source addresses from 92412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the descriptor 92512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 92612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 ppc440spe_desc_get_src_num(struct ppc440spe_adma_desc_slot *desc, 92712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 92812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 92912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 93012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xor_hw_desc; 93112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 93212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 93312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 93412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 93512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 93612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 93712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (dma_hw_desc->opc) { 93812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_NO_OP: 93912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DFILL128: 94012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 94112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DCHECK128: 94212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 94312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MV_SG1_SG2: 94412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MULTICAST: 94512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 94612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Only for RXOR operations we have more than 94712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * one source 94812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 94912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (le32_to_cpu(dma_hw_desc->sg1u) & 95012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_WIN_MSK) { 95112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR op, there are 2 or 3 sources */ 95212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (((le32_to_cpu(dma_hw_desc->sg1u) >> 95312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF) & 95412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_MSK) == DMA_RXOR12) { 95512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR 1-2 */ 95612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 2; 95712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 95812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR 1-2-3/1-2-4/1-2-5 */ 95912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 3; 96012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 96112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 96212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 96312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 96412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: unknown OPC 0x%02x\n", 96512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dma_hw_desc->opc); 96612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 96712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 96812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 96912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* up to 16 sources */ 97012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_hw_desc = desc->hw_desc; 97112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return xor_hw_desc->cbc & XOR_CDCR_OAC_MSK; 97212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 97312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 97412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 97512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 97612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 97712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 97812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 97912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_get_dst_num - get the number of destination addresses in 98012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * this descriptor 98112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 98212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 ppc440spe_desc_get_dst_num(struct ppc440spe_adma_desc_slot *desc, 98312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 98412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 98512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *dma_hw_desc; 98612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 98712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 98812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 98912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 99012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* May be 1 or 2 destinations */ 99112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_hw_desc = desc->hw_desc; 99212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (dma_hw_desc->opc) { 99312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_NO_OP: 99412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DCHECK128: 99512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 99612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MV_SG1_SG2: 99712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_DFILL128: 99812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 99912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_CDB_OPC_MULTICAST: 100012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (desc->dst_cnt == 2) 100112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 2; 100212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 100312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 100412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 100512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: unknown OPC 0x%02x\n", 100612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dma_hw_desc->opc); 100712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 100812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 100912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 101012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Always only 1 destination */ 101112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 101212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 101312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 101412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 101512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 101612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 101712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 101812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 101912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_get_link - get the address of the descriptor that 102012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * follows this one 102112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 102212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc, 102312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 102412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 102512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!desc->hw_next) 102612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 102712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 102812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return desc->hw_next->phys; 102912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 103012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 103112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 103212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_is_aligned - check alignment 103312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 103412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline int ppc440spe_desc_is_aligned( 103512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, int num_slots) 103612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 103712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return (desc->idx & (num_slots - 1)) ? 0 : 1; 103812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 103912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 104012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 104112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_xor_slot_count - get the number of slots necessary for 104212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * XOR operation 104312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 104412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt, 104512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int *slots_per_op) 104612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 104712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 104812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 104912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* each XOR descriptor provides up to 16 source operands */ 105012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS; 105112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 105212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT)) 105312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return slot_cnt; 105412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 105512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "%s: len %d > max %d !!\n", 105612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 105712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 105812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return slot_cnt; 105912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 106012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 106112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 106212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for 106312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * DMA2 PQ operation 106412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 106512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs, 106612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, size_t len) 106712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 106812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin signed long long order = 0; 106912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int state = 0; 107012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int addr_count = 0; 107112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 107212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 1; i < src_cnt; i++) { 107312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t cur_addr = srcs[i]; 107412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t old_addr = srcs[i-1]; 107512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (state) { 107612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 0: 107712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cur_addr == old_addr + len) { 107812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct RXOR */ 107912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 1; 108012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 1; 108112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 108212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 108312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (old_addr == cur_addr + len) { 108412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse RXOR */ 108512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = -1; 108612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 1; 108712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 108812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 108912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 109012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 3; 109112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 109212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 109312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 1: 109412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-2 || (order == -1 109512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin && cur_addr != old_addr - len)) { 109612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 109712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 109812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 109912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cur_addr == old_addr + len*order) { 110012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 2; 110112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 110212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 110312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cur_addr == old_addr + 2*len) { 110412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 2; 110512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 110612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 110712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cur_addr == old_addr + 3*len) { 110812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 2; 110912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i == src_cnt-1) 111012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 111112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 111212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 111312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 111412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 111512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 111612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 111712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 2: 111812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 111912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 112012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr_count++; 112112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 112212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 112312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (state == 3) 112412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 112512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 112612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt <= 1 || (state != 1 && state != 2)) { 112712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n", 112812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, src_cnt, state, addr_count, order); 112912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) 113012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("\t[%d] 0x%llx \n", i, srcs[i]); 113112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 113212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 113312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 113412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS; 113512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 113612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 113712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 113812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/****************************************************************************** 113912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ADMA channel low-level routines 114012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ******************************************************************************/ 114112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 114212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 114312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan); 114412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan); 114512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 114612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 114712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine 114812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 114912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_device_clear_eot_status( 115012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 115112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 115212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_regs *dma_reg; 115312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 115412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 *p = chan->device->dma_desc_pool_virt; 115512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *cdb; 115612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 rv, i; 115712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 115812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 115912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 116012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 116112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* read FIFO to ack */ 116212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_reg = chan->device->dma_reg; 116312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while ((rv = ioread32(&dma_reg->csfpl))) { 116412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i = rv & DMA_CDB_ADDR_MSK; 116512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb = (struct dma_cdb *)&p[i - 116612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (u32)chan->device->dma_desc_pool]; 116712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 116812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Clear opcode to ack. This is necessary for 116912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ZeroSum operations only 117012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 117112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cdb->opc = 0; 117212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 117312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_RXOR_RUN, 117412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &ppc440spe_rxor_state)) { 117512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* probably this is a completed RXOR op, 117612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * get pointer to CDB using the fact that 117712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * physical and virtual addresses of CDB 117812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * in pools have the same offsets 117912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 118012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (le32_to_cpu(cdb->sg1u) & 118112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE) { 118212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is a RXOR */ 118312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_RXOR_RUN, 118412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &ppc440spe_rxor_state); 118512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 118612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 118712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 118812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rv & DMA_CDB_STATUS_MSK) { 118912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* ZeroSum check failed 119012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 119112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 119212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t phys = rv & ~DMA_CDB_MSK; 119312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 119412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 119512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Update the status of corresponding 119612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor. 119712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 119812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &chan->chain, 119912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 120012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->phys == phys) 120112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 120212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 120312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 120412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * if cannot find the corresponding 120512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * slot it's a bug 120612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 120712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(&iter->chain_node == &chan->chain); 120812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 120912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->xor_check_result) { 121012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_PCHECK, 121112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags)) { 121212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *iter->xor_check_result |= 121312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin SUM_CHECK_P_RESULT; 121412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 121512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_QCHECK, 121612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags)) { 121712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *iter->xor_check_result |= 121812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin SUM_CHECK_Q_RESULT; 121912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 122012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 122112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 122212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 122312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 122412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 122512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rv = ioread32(&dma_reg->dsts); 122612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rv) { 122712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("DMA%d err status: 0x%x\n", 122812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id, rv); 122912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* write back to clear */ 123012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(rv, &dma_reg->dsts); 123112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 123212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 123312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 123412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reset status bits to ack */ 123512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 123612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rv = ioread32be(&xor_reg->sr); 123712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(rv, &xor_reg->sr); 123812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 123912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) { 124012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rv & XOR_IE_RPTIE_BIT) { 124112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Read PLB Timeout Error. 124212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Try to resubmit the CB 124312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 124412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 val = ioread32be(&xor_reg->ccbalr); 124512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 124612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(val, &xor_reg->cblalr); 124712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 124812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin val = ioread32be(&xor_reg->crsr); 124912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(val | XOR_CRSR_XAE_BIT, 125012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &xor_reg->crsr); 125112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 125212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("XOR ERR 0x%x status\n", rv); 125312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 125412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 125512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 125612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* if the XORcore is idle, but there are unprocessed CBs 125712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * then refetch the s/w chain here 125812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 125912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) && 126012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin do_xor_refetch) 126112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_append(chan); 126212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 126312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 126412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 126512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 126612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 126712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_is_busy - get the channel status 126812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 126912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan) 127012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 127112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_regs *dma_reg; 127212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 127312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int busy = 0; 127412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 127512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 127612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 127712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 127812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_reg = chan->device->dma_reg; 127912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* if command FIFO's head and tail pointers are equal and 128012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * status tail is the same as command, then channel is free 128112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 128212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) || 128312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp)) 128412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin busy = 1; 128512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 128612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 128712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* use the special status bit for the XORcore 128812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 128912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 129012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0; 129112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 129212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 129312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 129412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return busy; 129512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 129612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 129712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 129812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_set_first_xor_descriptor - init XORcore chain 129912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 130012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_set_first_xor_descriptor( 130112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 130212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *next_desc) 130312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 130412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg = chan->device->xor_reg; 130512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 130612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) 130712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_INFO "%s: Warn: XORcore is running " 130812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "when try to set the first CDB!\n", 130912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 131012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 131112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_submit = xor_last_linked = next_desc; 131212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 131312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr); 131412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 131512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(next_desc->phys, &xor_reg->cblalr); 131612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(0, &xor_reg->cblahr); 131712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT, 131812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &xor_reg->cbcr); 131912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 132012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->hw_chain_inited = 1; 132112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 132212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 132312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 132412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO. 132512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * called with irqs disabled 132612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 132712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan, 132812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc) 132912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 133012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 pcdb; 133112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_regs *dma_reg = chan->device->dma_reg; 133212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 133312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pcdb = desc->phys; 133412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_INT, &desc->flags)) 133512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pcdb |= DMA_CDB_NO_INT; 133612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 133712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan_last_sub[chan->device->id] = desc; 133812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 133912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(print_cb(chan, desc->hw_desc)); 134012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 134112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(pcdb, &dma_reg->cpfpl); 134212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 134312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 134412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 134512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_append - update the h/w chain in the channel 134612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 134712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan) 134812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 134912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 135012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 135112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb; 135212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 cur_desc; 135312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long flags; 135412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 135512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin local_irq_save(flags); 135612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 135712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 135812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 135912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 136012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cur_desc = ppc440spe_chan_get_current_descriptor(chan); 136112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 136212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(cur_desc)) { 136312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = chan_last_sub[chan->device->id]; 136412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!iter); 136512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 136612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* first peer */ 136712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = chan_first_cdb[chan->device->id]; 136812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!iter); 136912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_dma_put_desc(chan, iter); 137012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->hw_chain_inited = 1; 137112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 137212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 137312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* is there something new to append */ 137412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!iter->hw_next) 137512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 137612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 137712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* flush descriptors from the s/w queue to fifo */ 137812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_continue(iter, &chan->chain, chain_node) { 137912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_dma_put_desc(chan, iter); 138012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!iter->hw_next) 138112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 138212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 138312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 138412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 138512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* update h/w links and refetch */ 138612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!xor_last_submit->hw_next) 138712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 138812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 138912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 139012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* the last linked CDB has to generate an interrupt 139112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * that we'd be able to append the next lists to h/w 139212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * regardless of the XOR engine state at the moment of 139312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * appending of these next lists 139412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 139512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb = xor_last_linked->hw_desc; 139612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->cbc |= XOR_CBCR_CBCE_BIT; 139712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 139812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) { 139912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* XORcore is idle. Refetch now */ 140012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin do_xor_refetch = 0; 140112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_xor_set_link(xor_last_submit, 140212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_submit->hw_next); 140312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 140412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(print_cb_list(chan, 140512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_submit->hw_next)); 140612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 140712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_last_submit = xor_last_linked; 140812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(ioread32be(&xor_reg->crsr) | 140912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT, 141012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &xor_reg->crsr); 141112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 141212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* XORcore is running. Refetch later in the handler */ 141312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin do_xor_refetch = 1; 141412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 141512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 141612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 141712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 141812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 141912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin local_irq_restore(flags); 142012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 142112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 142212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 142312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor 142412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 142512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic u32 142612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan) 142712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 142812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_regs *dma_reg; 142912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 143012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 143112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!chan->hw_chain_inited)) 143212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* h/w descriptor chain is not initialized yet */ 143312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 143412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 143512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 143612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 143712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 143812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_reg = chan->device->dma_reg; 143912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK); 144012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 144112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 144212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ioread32be(&xor_reg->ccbalr); 144312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 144412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 144512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 144612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 144712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 144812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_run - enable the channel 144912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 145012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan) 145112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 145212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_regs *xor_reg; 145312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 145412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 145512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 145612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 145712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMAs are always enabled, do nothing */ 145812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 145912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 146012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* drain write buffer */ 146112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xor_reg = chan->device->xor_reg; 146212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 146312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* fetch descriptor pointed to in <link> */ 146412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT, 146512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &xor_reg->crsr); 146612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 146712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 146812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 146912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 147012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/****************************************************************************** 147112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ADMA device level 147212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ******************************************************************************/ 147312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 147412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan); 147512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan); 147612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 147712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic dma_cookie_t 147812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx); 147912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 148012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx, 148112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index); 148212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 148312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx, 148412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index); 148512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 148612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 148712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx, 148812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *paddr, unsigned long flags); 148912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 149012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx, 149112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index); 149212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 149312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx, 149412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned char mult, int index, int dst_pos); 149512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void 149612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx, 149712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t paddr, dma_addr_t qaddr); 149812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 149912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct page *ppc440spe_rxor_srcs[32]; 150012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 150112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 150212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_can_rxor - check if the operands may be processed with RXOR 150312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 150412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len) 150512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 150612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i, order = 0, state = 0; 150712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int idx = 0; 150812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 150912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!(src_cnt > 1))) 151012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 151112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 151212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs)); 151312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 151412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Skip holes in the source list before checking */ 151512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) { 151612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!srcs[i]) 151712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 151812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_srcs[idx++] = srcs[i]; 151912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 152012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt = idx; 152112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 152212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 1; i < src_cnt; i++) { 152312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin char *cur_addr = page_address(ppc440spe_rxor_srcs[i]); 152412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]); 152512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 152612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (state) { 152712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 0: 152812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cur_addr == old_addr + len) { 152912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct RXOR */ 153012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 1; 153112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 1; 153212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (old_addr == cur_addr + len) { 153312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse RXOR */ 153412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = -1; 153512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 1; 153612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 153712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out; 153812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 153912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 1: 154012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((i == src_cnt - 2) || 154112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (order == -1 && cur_addr != old_addr - len)) { 154212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 154312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 154412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if ((cur_addr == old_addr + len * order) || 154512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (cur_addr == old_addr + 2 * len) || 154612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (cur_addr == old_addr + 3 * len)) { 154712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 2; 154812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 154912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 155012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 155112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 155212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 155312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 2: 155412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin order = 0; 155512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin state = 0; 155612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 155712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 155812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 155912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 156012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout: 156112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (state == 1 || state == 2) 156212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 156312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 156412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 156512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 156612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 156712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 156812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_device_estimate - estimate the efficiency of processing 156912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the operation given on this channel. It's assumed that 'chan' is 157012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * capable to process 'cap' type of operation. 157112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @chan: channel to use 157212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @cap: type of transaction 157312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @dst_lst: array of destination pointers 157412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @dst_cnt: number of destination operands 157512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @src_lst: array of source pointers 157612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @src_cnt: number of source operands 157712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @src_sz: size of each source operand 157812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 157912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_estimate(struct dma_chan *chan, 158012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt, 158112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct page **src_lst, int src_cnt, size_t src_sz) 158212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 158312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int ef = 1; 158412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 158512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cap == DMA_PQ || cap == DMA_PQ_VAL) { 158612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* If RAID-6 capabilities were not activated don't try 158712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * to use them 158812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 158912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!ppc440spe_r6_enabled)) 159012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -1; 159112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 159212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* In the current implementation of ppc440spe ADMA driver it 159312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * makes sense to pick out only pq case, because it may be 159412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * processed: 159512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (1) either using Biskup method on DMA2; 159612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (2) or on DMA0/1. 159712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Thus we give a favour to (1) if the sources are suitable; 159812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * else let it be processed on one of the DMA0/1 engines. 159912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * In the sum_product case where destination is also the 160012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * source process it on DMA0/1 only. 160112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 160212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) { 160312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 160412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1]) 160512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ef = 0; /* sum_product case, process on DMA0/1 */ 160612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz)) 160712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ef = 3; /* override (DMA0/1 + idle) */ 160812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 160912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ef = 0; /* can't process on DMA2 if !rxor */ 161012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 161112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 161212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* channel idleness increases the priority */ 161312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(ef) && 161412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan))) 161512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ef++; 161612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 161712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ef; 161812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 161912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 162012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstruct dma_chan * 162112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap, 162212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct page **dst_lst, int dst_cnt, struct page **src_lst, 162312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, size_t src_sz) 162412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 162512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *best_chan = NULL; 162612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc_dma_chan_ref *ref; 162712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int best_rank = -1; 162812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 162912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!src_sz)) 163012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 163112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_sz > PAGE_SIZE) { 163212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 163312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * should a user of the api ever pass > PAGE_SIZE requests 163412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * we sort out cases where temporary page-sized buffers 163512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * are used. 163612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 163712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (cap) { 163812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_PQ: 163912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt == 1 && dst_lst[1] == src_lst[0]) 164012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 164112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt == 2 && dst_lst[1] == src_lst[1]) 164212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 164312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 164412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_PQ_VAL: 164512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case DMA_XOR_VAL: 164612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 164712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 164812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 164912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 165012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 165112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 165212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) { 165312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(cap, ref->chan->device->cap_mask)) { 165412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int rank; 165512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 165612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst, 165712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst_cnt, src_lst, src_cnt, src_sz); 165812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (rank > best_rank) { 165912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin best_rank = rank; 166012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin best_chan = ref->chan; 166112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 166212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 166312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 166412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 166512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return best_chan; 166612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 166712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinEXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel); 166812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 166912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 167012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_get_group_entry - get group entry with index idx 167112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @tdesc: is the last allocated slot in the group. 167212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 167312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot * 167412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx) 167512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 167612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter = tdesc->group_head; 167712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i = 0; 167812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 167912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) { 168012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n", 168112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt); 168212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 168312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 168412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 168512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &tdesc->group_list, chain_node) { 168612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i++ == entry_idx) 168712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 168812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 168912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return iter; 169012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 169112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 169212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 169312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_free_slots - flags descriptor slots for reuse 169412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @slot: Slot to free 169512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Caller must hold &ppc440spe_chan->lock while calling this function 169612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 169712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot, 169812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 169912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 170012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int stride = slot->slots_per_op; 170112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 170212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (stride--) { 170312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->slots_per_op = 0; 170412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot = list_entry(slot->slot_node.next, 170512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 170612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node); 170712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 170812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 170912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 171012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_unmap(struct ppc440spe_adma_chan *chan, 171112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc) 171212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 171312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 src_cnt, dst_cnt; 171412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr; 171512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 171612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 171712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * get the number of sources & destination 171812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * included in this descriptor and unmap 171912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * them all 172012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 172112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt = ppc440spe_desc_get_src_num(desc, chan); 172212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst_cnt = ppc440spe_desc_get_dst_num(desc, chan); 172312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 172412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* unmap destinations */ 172512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(desc->async_tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) { 172612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (dst_cnt--) { 172712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr = ppc440spe_desc_get_dest_addr( 172812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, chan, dst_cnt); 172912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(chan->device->dev, 173012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr, desc->unmap_len, 173112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_FROM_DEVICE); 173212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 173312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 173412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 173512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* unmap sources */ 173612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(desc->async_tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) { 173712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (src_cnt--) { 173812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr = ppc440spe_desc_get_src_addr( 173912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, chan, src_cnt); 174012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(chan->device->dev, 174112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr, desc->unmap_len, 174212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_TO_DEVICE); 174312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 174412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 174512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 174612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 174712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 174812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_run_tx_complete_actions - call functions to be called 174912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * upon completion 175012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 175112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic dma_cookie_t ppc440spe_adma_run_tx_complete_actions( 175212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 175312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 175412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t cookie) 175512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 175612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 175712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 175812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(desc->async_tx.cookie < 0); 175912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (desc->async_tx.cookie > 0) { 176012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = desc->async_tx.cookie; 176112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->async_tx.cookie = 0; 176212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 176312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* call the callback (must not sleep or submit new 176412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * operations to this channel) 176512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 176612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (desc->async_tx.callback) 176712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->async_tx.callback( 176812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->async_tx.callback_param); 176912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 177012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* unmap dma addresses 177112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (unmap_single vs unmap_page?) 177212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 177312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * actually, ppc's dma_unmap_page() functions are empty, so 177412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the following code is just for the sake of completeness 177512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 177612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan && chan->needs_unmap && desc->group_head && 177712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->unmap_len) { 177812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *unmap = 177912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->group_head; 178012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* assume 1 slot per op always */ 178112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 slot_count = unmap->slot_cnt; 178212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 178312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Run through the group list and unmap addresses */ 178412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < slot_count; i++) { 178512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!unmap); 178612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_unmap(chan, unmap); 178712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unmap = unmap->hw_next; 178812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 178912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 179012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 179112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 179212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* run dependent operations */ 179312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_run_dependencies(&desc->async_tx); 179412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 179512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return cookie; 179612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 179712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 179812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 179912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set) 180012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 180112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc, 180212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 180312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 180412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* the client is allowed to attach dependent operations 180512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * until 'ack' is set 180612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 180712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!async_tx_test_ack(&desc->async_tx)) 180812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 180912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 181012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* leave the last descriptor in the chain 181112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * so we can append to it 181212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 181312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (list_is_last(&desc->chain_node, &chan->chain) || 181412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->phys == ppc440spe_chan_get_current_descriptor(chan)) 181512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 181612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 181712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan->device->id != PPC440SPE_XOR_ID) { 181812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* our DMA interrupt handler clears opc field of 181912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * each processed descriptor. For all types of 182012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * operations except for ZeroSum we do not actually 182112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * need ack from the interrupt handler. ZeroSum is a 182212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * special case since the result of this operation 182312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * is available from the handler only, so if we see 182412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * such type of descriptor (which is unprocessed yet) 182512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * then leave it in chain. 182612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 182712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *cdb = desc->hw_desc; 182812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cdb->opc == DMA_CDB_OPC_DCHECK128) 182912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 1; 183012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 183112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 183212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n", 183312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->phys, desc->idx, desc->slots_per_op); 183412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 183512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&desc->chain_node); 183612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_free_slots(desc, chan); 183712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 183812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 183912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 184012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 184112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine 184212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * which runs through the channel CDBs list until reach the descriptor 184312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * currently processed. When routine determines that all CDBs of group 184412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * are completed then corresponding callbacks (if any) are called and slots 184512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * are freed. 184612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 184712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan) 184812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 184912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL; 185012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t cookie = 0; 185112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 current_desc = ppc440spe_chan_get_current_descriptor(chan); 185212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int busy = ppc440spe_chan_is_busy(chan); 185312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int seen_current = 0, slot_cnt = 0, slots_per_op = 0; 185412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 185512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n", 185612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id, __func__); 185712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 185812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!current_desc) { 185912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* There were no transactions yet, so 186012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * nothing to clean 186112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 186212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return; 186312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 186412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 186512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* free completed slots from the chain starting with 186612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the oldest descriptor 186712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 186812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(iter, _iter, &chan->chain, 186912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 187012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d " 187112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "busy: %d this_desc: %#llx next_desc: %#x " 187212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "cur: %#x ack: %d\n", 187312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->async_tx.cookie, iter->idx, busy, iter->phys, 187412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_get_link(iter, chan), current_desc, 187512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin async_tx_test_ack(&iter->async_tx)); 187612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prefetch(_iter); 187712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prefetch(&_iter->async_tx); 187812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 187912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do not advance past the current descriptor loaded into the 188012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * hardware channel,subsequent descriptors are either in process 188112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * or have not been submitted 188212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 188312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (seen_current) 188412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 188512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 188612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* stop the search if we reach the current descriptor and the 188712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * channel is busy, or if it appears that the current descriptor 188812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * needs to be re-read (i.e. has been appended to) 188912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 189012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->phys == current_desc) { 189112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(seen_current++); 189212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (busy || ppc440spe_desc_get_link(iter, chan)) { 189312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* not all descriptors of the group have 189412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * been completed; exit. 189512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 189612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 189712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 189812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 189912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 190012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* detect the start of a group transaction */ 190112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!slot_cnt && !slots_per_op) { 190212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = iter->slot_cnt; 190312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = iter->slots_per_op; 190412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slot_cnt <= slots_per_op) { 190512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = 0; 190612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = 0; 190712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 190812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 190912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 191012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slot_cnt) { 191112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!group_start) 191212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = iter; 191312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt -= slots_per_op; 191412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 191512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 191612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* all the members of a group are complete */ 191712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slots_per_op != 0 && slot_cnt == 0) { 191812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter; 191912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int end_of_chain = 0; 192012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 192112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* clean up the group */ 192212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = group_start->slot_cnt; 192312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin grp_iter = group_start; 192412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe_from(grp_iter, _grp_iter, 192512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &chan->chain, chain_node) { 192612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 192712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = ppc440spe_adma_run_tx_complete_actions( 192812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin grp_iter, chan, cookie); 192912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 193012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt -= slots_per_op; 193112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin end_of_chain = ppc440spe_adma_clean_slot( 193212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin grp_iter, chan); 193312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (end_of_chain && slot_cnt) { 193412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Should wait for ZeroSum completion */ 193512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cookie > 0) 19364d4e58de32a192fea65ab84509d17d199bd291c8Russell King - ARM Linux chan->common.completed_cookie = cookie; 193712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return; 193812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 193912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 194012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slot_cnt == 0 || end_of_chain) 194112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 194212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 194312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 194412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* the group should be complete at this point */ 194512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(slot_cnt); 194612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 194712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = 0; 194812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = NULL; 194912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (end_of_chain) 195012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 195112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 195212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 195312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (slots_per_op) /* wait for group completion */ 195412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 195512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 195612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan, 195712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cookie); 195812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 195912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_adma_clean_slot(iter, chan)) 196012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 196112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 196212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 196312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!seen_current); 196412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 196512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (cookie > 0) { 19664d4e58de32a192fea65ab84509d17d199bd291c8Russell King - ARM Linux chan->common.completed_cookie = cookie; 196712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("\tcompleted cookie %d\n", cookie); 196812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 196912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 197012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 197112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 197212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 197312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_tasklet - clean up watch-dog initiator 197412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 197512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_tasklet(unsigned long data) 197612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 197712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan = (struct ppc440spe_adma_chan *) data; 197812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 197912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING); 198012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __ppc440spe_adma_slot_cleanup(chan); 198112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock(&chan->lock); 198212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 198312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 198412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 198512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_slot_cleanup - clean up scheduled initiator 198612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 198712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan) 198812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 198912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&chan->lock); 199012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __ppc440spe_adma_slot_cleanup(chan); 199112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 199212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 199312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 199412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 199512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_alloc_slots - allocate free slots (if any) 199612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 199712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots( 199812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, int num_slots, 199912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slots_per_op) 200012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 200112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter = NULL, *_iter; 200212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *alloc_start = NULL; 200312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct list_head chain = LIST_HEAD_INIT(chain); 200412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slots_found, retry = 0; 200512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 200612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 200712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!num_slots || !slots_per_op); 200812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* start search from the last allocated descrtiptor 200912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * if a contiguous allocation can not be found start searching 201012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * from the beginning of the list 201112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 201212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinretry: 201312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_found = 0; 201412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (retry == 0) 201512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = chan->last_used; 201612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 201712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(&chan->all_slots, 201812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 201912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node); 202012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots, 202112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node) { 202212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prefetch(_iter); 202312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin prefetch(&_iter->async_tx); 202412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->slots_per_op) { 202512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_found = 0; 202612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 202712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 202812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 202912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* start the allocation if the slot is correctly aligned */ 203012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!slots_found++) 203112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin alloc_start = iter; 203212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 203312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slots_found == num_slots) { 203412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *alloc_tail = NULL; 203512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *last_used = NULL; 203612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 203712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = alloc_start; 203812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (num_slots) { 203912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 204012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* pre-ack all but the last descriptor */ 204112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (num_slots != slots_per_op) 204212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin async_tx_ack(&iter->async_tx); 204312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 204412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_add_tail(&iter->chain_node, &chain); 204512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin alloc_tail = iter; 204612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->async_tx.cookie = 0; 204712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 204812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->flags = 0; 204912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->slot_cnt = num_slots; 205012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->xor_check_result = NULL; 205112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < slots_per_op; i++) { 205212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->slots_per_op = slots_per_op - i; 205312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin last_used = iter; 205412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->slot_node.next, 205512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 205612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node); 205712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 205812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin num_slots -= slots_per_op; 205912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 206012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin alloc_tail->group_head = alloc_start; 206112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin alloc_tail->async_tx.cookie = -EBUSY; 206212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_splice(&chain, &alloc_tail->group_list); 206312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->last_used = last_used; 206412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return alloc_tail; 206512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 206612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 206712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!retry++) 206812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto retry; 206912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 207012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* try to free some slots if the allocation fails */ 207112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_schedule(&chan->irq_tasklet); 207212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 207312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 207412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 207512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 207612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_alloc_chan_resources - allocate pools for CDB slots 207712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 207812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan) 207912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 208012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 208112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *slot = NULL; 208212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin char *hw_desc; 208312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i, db_sz; 208412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int init; 208512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 208612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 208712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin init = ppc440spe_chan->slots_allocated ? 0 : 1; 208812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->chan_id = ppc440spe_chan->device->id; 208912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 209012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Allocate descriptor slots */ 209112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i = ppc440spe_chan->slots_allocated; 209212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID) 209312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin db_sz = sizeof(struct dma_cdb); 209412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 209512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin db_sz = sizeof(struct xor_cb); 209612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 209712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) { 209812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot), 209912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin GFP_KERNEL); 210012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!slot) { 210112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_INFO "SPE ADMA Channel only initialized" 210212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " %d descriptor slots", i--); 210312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 210412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 210512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 210612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt; 210712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->hw_desc = (void *) &hw_desc[i * db_sz]; 210812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_async_tx_descriptor_init(&slot->async_tx, chan); 210912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->async_tx.tx_submit = ppc440spe_adma_tx_submit; 211012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&slot->chain_node); 211112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&slot->slot_node); 211212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&slot->group_list); 211312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz; 211412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot->idx = i; 211512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 211612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 211712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->slots_allocated++; 211812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots); 211912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 212012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 212112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 212212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (i && !ppc440spe_chan->last_used) { 212312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->last_used = 212412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_entry(ppc440spe_chan->all_slots.next, 212512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 212612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_node); 212712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 212812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 212912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 213012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: allocated %d descriptor slots\n", 213112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, i); 213212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 213312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* initialize the channel and the chain with a null operation */ 213412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (init) { 213512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (ppc440spe_chan->device->id) { 213612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 213712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 213812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->hw_chain_inited = 0; 213912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Use WXOR for self-testing */ 214012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!ppc440spe_r6_tchan) 214112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_r6_tchan = ppc440spe_chan; 214212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 214312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 214412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_start_null_xor(ppc440spe_chan); 214512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 214612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin default: 214712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 214812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 214912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->needs_unmap = 1; 215012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 215112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 215212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return (i > 0) ? i : -ENOMEM; 215312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 215412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 215512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 215612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_rxor_set_region_data - 215712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 215812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc, 215912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 xor_arg_no, u32 mask) 216012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 216112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 216212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 216312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->ops[xor_arg_no].h |= mask; 216412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 216512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 216612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 216712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_rxor_set_src - 216812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 216912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc, 217012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 xor_arg_no, dma_addr_t addr) 217112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 217212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 217312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 217412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE; 217512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->ops[xor_arg_no].l = addr; 217612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 217712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 217812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 217912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_rxor_set_mult - 218012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 218112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc, 218212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u8 xor_arg_no, u8 idx, u8 mult) 218312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 218412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 218512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 218612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8); 218712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 218812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 218912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 219012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold 219112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * has been achieved 219212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 219312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan) 219412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 219512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n", 219612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id, chan->pending); 219712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 219812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) { 219912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->pending = 0; 220012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_append(chan); 220112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 220212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 220312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 220412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 220512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_tx_submit - submit new descriptor group to the channel 220612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (it's not necessary that descriptors will be submitted to the h/w 220712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * chains too right now) 220812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 220912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx) 221012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 221112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc; 221212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan); 221312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail; 221412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 221512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slots_per_op; 221612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t cookie; 221712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 221812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = tx_to_ppc440spe_adma_slot(tx); 221912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 222012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 222112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = group_start->slot_cnt; 222212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = group_start->slots_per_op; 222312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 222412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&chan->lock); 2225884485e1f12dcd39390f042e772cdbefc9ebb750Russell King - ARM Linux cookie = dma_cookie_assign(tx); 222612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 222712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(list_empty(&chan->chain))) { 222812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* first peer */ 222912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_splice_init(&sw_desc->group_list, &chan->chain); 223012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan_first_cdb[chan->device->id] = group_start; 223112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 223212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* isn't first peer, bind CDBs to chain */ 223312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin old_chain_tail = list_entry(chan->chain.prev, 223412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 223512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 223612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_splice_init(&sw_desc->group_list, 223712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &old_chain_tail->chain_node); 223812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* fix up the hardware chain */ 223912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_link(chan, old_chain_tail, group_start); 224012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 224112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 224212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* increment the pending count by the number of operations */ 224312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->pending += slot_cnt / slots_per_op; 224412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_check_threshold(chan); 224512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 224612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 224712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, 224812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n", 224912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id, __func__, 225012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.cookie, sw_desc->idx, sw_desc); 225112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 225212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return cookie; 225312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 225412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 225512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 225612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation 225712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 225812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt( 225912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, unsigned long flags) 226012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 226112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 226212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 226312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op; 226412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 226512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 226612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 226712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 226812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id, 226912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 227012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 227112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 227212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = slots_per_op = 1; 227312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 227412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op); 227512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 227612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 227712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan); 227812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start->unmap_len = 0; 227912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 228012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 228112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 228212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 228312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 228412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 228512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 228612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 228712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation 228812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 228912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy( 229012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t dma_dest, 229112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t dma_src, size_t len, unsigned long flags) 229212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 229312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 229412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 229512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op; 229612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 229712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 229812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 229912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!len)) 230012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 230112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 2302427cdf19b97e509e21e5d347e18d8b0b34723dfcColy Li BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT); 230312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 230412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 230512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 230612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 230712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s len: %u int_en %d\n", 230812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, __func__, len, 230912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags & DMA_PREP_INTERRUPT ? 1 : 0); 231012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = slots_per_op = 1; 231112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 231212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op); 231312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 231412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 231512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_memcpy(group_start, flags); 231612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_set_dest(group_start, dma_dest, 0); 231712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0); 231812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len); 231912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->unmap_len = len; 232012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 232112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 232212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 232312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 232412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 232512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 232612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 232712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 232812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation 232912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 233012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor( 233112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t dma_dest, 233212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dma_src, u32 src_cnt, size_t len, 233312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long flags) 233412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 233512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 233612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 233712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op; 233812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 233912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 234012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 234112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id, 234212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_dest, dma_src, src_cnt)); 234312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(!len)) 234412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 2345427cdf19b97e509e21e5d347e18d8b0b34723dfcColy Li BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 234612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 234712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 234812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n", 234912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, __func__, src_cnt, len, 235012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags & DMA_PREP_INTERRUPT ? 1 : 0); 235112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 235212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 235312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op); 235412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 235512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op); 235612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 235712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 235812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_xor(group_start, src_cnt, flags); 235912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_set_dest(group_start, dma_dest, 0); 236012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (src_cnt--) 236112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_memcpy_xor_set_src(group_start, 236212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_src[src_cnt], src_cnt); 236312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len); 236412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->unmap_len = len; 236512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 236612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 236712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 236812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 236912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 237012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 237112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 237212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline void 237312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc, 237412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt); 237512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor); 237612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 237712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 237812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_init_dma2rxor_slot - 237912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 238012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_init_dma2rxor_slot( 238112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 238212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *src, int src_cnt) 238312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 238412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 238512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 238612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* initialize CDB */ 238712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < src_cnt; i++) { 238812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i, 238912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->src_cnt, (u32)src[i]); 239012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 239112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 239212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 239312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 239412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma01_prep_mult - 239512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for Q operation where destination is also the source 239612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 239712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult( 239812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan, 239912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 240012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf, size_t len, unsigned long flags) 240112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 240212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL; 240312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 240412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 240512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 240612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_WXOR, &op); 240712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = 2; 240812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 240912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 241012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 241112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* use WXOR, each descriptor occupies one slot */ 241212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 241312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 241412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 241512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 241612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 241712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 241812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 241912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bits(op, &sw_desc->flags); 242012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->src_cnt = src_cnt; 242112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt = dst_cnt; 242212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* First descriptor, zero data in the destination and copy it 242312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * to q page using MULTICAST transfer. 242412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 242512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&sw_desc->group_list, 242612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 242712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 242812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 242912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 243012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 243112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 243212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 243312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 243412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 243512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MULTICAST; 243612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 243712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 243812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, dst[0], 0); 243912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1); 244012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 244112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src[0]); 244212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 244312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 244412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 244512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 244612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Second descriptor, multiply data from the q page 244712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * and store the result in real destination. 244812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 244912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 245012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 245112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 245212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 245312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 245412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 245512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 245612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 245712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 245812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 245912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 246012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 246112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 246212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB, dst[1]); 246312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 246412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, dst[0], 0); 246512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 246612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 246712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1, scf[0]); 246812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 246912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 247012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 247112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 247212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 247312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 247412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 247512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc; 247612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 247712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 247812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 247912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma01_prep_sum_product - 248012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also 248112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the source. 248212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 248312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product( 248412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan, 248512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dst, dma_addr_t *src, int src_cnt, 248612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf, size_t len, unsigned long flags) 248712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 248812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL; 248912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 249012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 249112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 249212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_WXOR, &op); 249312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = 3; 249412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 249512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 249612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 249712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR, each descriptor occupies one slot */ 249812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 249912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 250012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 250112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 250212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 250312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 250412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 250512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bits(op, &sw_desc->flags); 250612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->src_cnt = src_cnt; 250712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt = 1; 250812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 1st descriptor, src[1] data to q page and zero destination */ 250912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&sw_desc->group_list, 251012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 251112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 251212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 251312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 251412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 251512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 251612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 251712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 251812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MULTICAST; 251912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 252012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 252112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *dst, 0); 252212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 0, 252312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->qdest, 1); 252412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 252512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src[1]); 252612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 252712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 252812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 252912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 2nd descriptor, multiply src[1] data and store the 253012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * result in destination */ 253112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 253212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 253312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 253412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 253512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 253612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 253712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 253812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 253912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 254012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 254112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 254212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 254312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 254412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 254512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 254612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 254712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->qdest); 254812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 254912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *dst, 0); 255012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 255112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1, scf[1]); 255212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 255312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 255412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 255512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 255612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * 3rd descriptor, multiply src[0] data and xor it 255712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * with destination 255812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 255912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&iter->chain_node, 256012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 256112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 256212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 256312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 256412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_INTERRUPT) 256512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_INT, &iter->flags); 256612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 256712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_INT, &iter->flags); 256812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 256912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 257012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 257112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 257212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src[0]); 257312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 257412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *dst, 0); 257512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 257612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1, scf[0]); 257712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 257812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 257912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 258012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 258112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 258212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 258312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 258412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc; 258512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 258612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 258712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq( 258812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan, 258912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 259012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf, size_t len, unsigned long flags) 259112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 259212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt; 259312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter; 259412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 259512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned char mult = 1; 259612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 259712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n", 259812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dst_cnt, src_cnt, len); 259912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* select operations WXOR/RXOR depending on the 260012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * source addresses of operators and the number 260112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * of destinations (RXOR support only Q-parity calculations) 260212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 260312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_WXOR, &op); 260412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) { 260512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* no active RXOR; 260612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * do RXOR if: 260712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - there are more than 1 source, 260812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - len is aligned on 512-byte boundary, 260912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - source addresses fit to one of 4 possible regions. 261012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 261112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt > 1 && 261212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin !(len & MQ0_CF2H_RXOR_BS_MASK) && 261312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (src[0] + len) == src[1]) { 261412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* may do RXOR R1 R2 */ 261512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR, &op); 261612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt != 2) { 261712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* may try to enhance region of RXOR */ 261812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((src[1] + len) == src[2]) { 261912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 R3 */ 262012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR123, 262112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &op); 262212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if ((src[1] + len * 2) == src[2]) { 262312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 R4 */ 262412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR124, &op); 262512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if ((src[1] + len * 3) == src[2]) { 262612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 R5 */ 262712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR125, 262812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &op); 262912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 263012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 */ 263112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR12, 263212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &op); 263312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 263412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 263512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* do RXOR R1 R2 */ 263612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_RXOR12, &op); 263712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 263812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 263912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 264012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_RXOR, &op)) { 264112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* can not do this operation with RXOR */ 264212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_RXOR_RUN, 264312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &ppc440spe_rxor_state); 264412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 264512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* can do; set block size right now */ 264612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_rxor_block_size(len); 264712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 264812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 264912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 265012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Number of necessary slots depends on operation type selected */ 265112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_RXOR, &op)) { 265212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is a WXOR only chain. Need descriptors for each 265312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * source to GF-XOR them with WXOR, and need descriptors 265412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for each destination to zero them with WXOR 265512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 265612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = src_cnt; 265712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 265812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_P) { 265912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt++; 266012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_P, &op); 266112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 266212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_Q) { 266312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt++; 266412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_Q, &op); 266512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 266612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 266712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Need 1/2 descriptor for RXOR operation, and 266812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * need (src_cnt - (2 or 3)) for WXOR of sources 266912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * remained (if any) 267012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 267112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = dst_cnt; 267212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 267312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_P) 267412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_P, &op); 267512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_Q) 267612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_Q, &op); 267712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 267812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_RXOR12, &op)) 267912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt += src_cnt - 2; 268012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 268112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt += src_cnt - 3; 268212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 268312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Thus we have either RXOR only chain or 268412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * mixed RXOR/WXOR 268512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 268612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (slot_cnt == dst_cnt) 268712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR only chain */ 268812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin clear_bit(PPC440SPE_DESC_WXOR, &op); 268912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 269012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 269112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 269212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* for both RXOR/WXOR each descriptor occupies one slot */ 269312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 269412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 269512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt, 269612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags, op); 269712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 269812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* setup dst/src/mult */ 269912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n", 270012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dst[0], dst[1]); 270112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_dest(sw_desc, dst, flags); 270212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (src_cnt--) { 270312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt], 270412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt); 270512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 270612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* NOTE: "Multi = 0 is equivalent to = 1" as it 270712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf 270812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * doesn't work for RXOR with DMA0/1! Instead, multi=0 270912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * leads to zeroing source data after RXOR. 271012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * So, for P case set-up mult=1 explicitly. 271112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 271212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 271312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult = scf[src_cnt]; 271412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src_mult(sw_desc, 271512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult, src_cnt, dst_cnt - 1); 271612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 271712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 271812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup byte count foreach slot just allocated */ 271912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 272012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, 272112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 272212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, 272312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan, len); 272412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 272512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 272612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 272712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 272812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 272912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc; 273012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 273112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 273212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq( 273312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan, 273412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 273512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const unsigned char *scf, size_t len, unsigned long flags) 273612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 273712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, descs_per_op; 273812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter; 273912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 274012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned char mult = 1; 274112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 274212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!dst_cnt); 274312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n", 274412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, dst_cnt, src_cnt, len);*/ 274512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 274612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 274712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len); 274812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (descs_per_op < 0) { 274912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 275012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return NULL; 275112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 275212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 275312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* depending on number of sources we have 1 or 2 RXOR chains */ 275412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = descs_per_op * dst_cnt; 275512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 275612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 275712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 275812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op = slot_cnt; 275912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 276012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 276112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt, 276212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin --op ? 0 : flags); 276312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 276412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len); 276512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 276612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 276712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_init_rxor_cursor(&(iter->rxor_cursor)); 276812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->rxor_cursor.len = len; 276912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->descs_per_op = descs_per_op; 277012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 277112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op = 0; 277212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 277312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op++; 277412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (op % descs_per_op == 0) 277512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_init_dma2rxor_slot(iter, src, 277612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt); 277712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(!list_is_last(&iter->chain_node, 277812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list))) { 277912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set 'next' pointer */ 278012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = 278112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_entry(iter->chain_node.next, 278212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 278312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 278412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_xor_set_link(iter, iter->hw_next); 278512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 278612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is the last descriptor. */ 278712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = NULL; 278812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 278912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 279012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 279112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* fixup head descriptor */ 279212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt = dst_cnt; 279312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_P) 279412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_P, &sw_desc->flags); 279512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_ZERO_Q) 279612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags); 279712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 279812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* setup dst/src/mult */ 279912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_dest(sw_desc, dst, flags); 280012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 280112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (src_cnt--) { 280212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* handle descriptors (if dst_cnt == 2) inside 280312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the ppc440spe_adma_pq_set_srcxxx() functions 280412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 280512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt], 280612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt); 280712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 280812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult = scf[src_cnt]; 280912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src_mult(sw_desc, 281012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult, src_cnt, dst_cnt - 1); 281112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 281212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 281312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 281412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_rxor_block_size(len); 281512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc; 281612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 281712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 281812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 281912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation 282012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 282112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq( 282212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, 282312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt, const unsigned char *scf, 282412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size_t len, unsigned long flags) 282512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 282612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 282712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc = NULL; 282812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int dst_cnt = 0; 282912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 283012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 283112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 283212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id, 283312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst, src, src_cnt)); 283412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!len); 2835427cdf19b97e509e21e5d347e18d8b0b34723dfcColy Li BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 283612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!src_cnt); 283712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 283812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt == 1 && dst[1] == src[0]) { 283912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t dest[2]; 284012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 284112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* dst[1] is real destination (Q) */ 284212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dest[0] = dst[1]; 284312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is the page to multicast source data to */ 284412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dest[1] = ppc440spe_chan->qdest; 284512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan, 284612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dest, 2, src, src_cnt, scf, len, flags); 284712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 284812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 284912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 285012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (src_cnt == 2 && dst[1] == src[1]) { 285112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan, 285212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &dst[1], src, 2, scf, len, flags); 285312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 285412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 285512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 285612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(flags & DMA_PREP_PQ_DISABLE_P)) { 285712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!dst[0]); 285812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst_cnt++; 285912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags |= DMA_PREP_ZERO_P; 286012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 286112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 286212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(flags & DMA_PREP_PQ_DISABLE_Q)) { 286312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!dst[1]); 286412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst_cnt++; 286512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags |= DMA_PREP_ZERO_Q; 286612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 286712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 286812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(!dst_cnt); 286912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 287012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 287112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n", 287212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, __func__, src_cnt, len, 287312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags & DMA_PREP_INTERRUPT ? 1 : 0); 287412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 287512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (ppc440spe_chan->device->id) { 287612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 287712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 287812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan, 287912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst, dst_cnt, src, src_cnt, scf, 288012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len, flags); 288112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 288212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 288312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 288412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan, 288512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dst, dst_cnt, src, src_cnt, scf, 288612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len, flags); 288712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 288812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 288912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 289012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 289112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 289212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 289312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 289412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for 289512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * a PQ_ZERO_SUM operation 289612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 289712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum( 289812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, 289912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int src_cnt, const unsigned char *scf, size_t len, 290012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin enum sum_check_flags *pqres, unsigned long flags) 290112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 290212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 290312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *iter; 290412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t pdest, qdest; 290512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op, idst, dst_cnt; 290612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 290712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 290812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 290912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_PQ_DISABLE_P) 291012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pdest = 0; 291112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 291212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pdest = pq[0]; 291312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 291412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_PQ_DISABLE_Q) 291512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qdest = 0; 291612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 291712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qdest = pq[1]; 291812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 291912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id, 292012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src, src_cnt, scf)); 292112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 292212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Always use WXOR for P/Q calculations (two destinations). 292312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Need 1 or 2 extra slots to verify results are zero. 292412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 292512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idst = dst_cnt = (pdest && qdest) ? 2 : 1; 292612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 292712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* One additional slot per destination to clone P/Q 292812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * before calculation (we have to preserve destinations). 292912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 293012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = src_cnt + dst_cnt * 2; 293112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op = 1; 293212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 293312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 293412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 293512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slots_per_op); 293612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 293712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt); 293812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 293912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup byte count for each slot just allocated */ 294012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.flags = flags; 294112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 294212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 294312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len); 294412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = len; 294512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 294612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 294712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (pdest) { 294812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 294912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 295012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 295112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = sw_desc->group_head; 295212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 295312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 295412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 295512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 295612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 295712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 295812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 295912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->src_cnt = 0; 296012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->dst_cnt = 0; 296112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 0, 296212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->pdest, 0); 296312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest); 296412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 296512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len); 296612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = 0; 296712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* override pdest to preserve original P */ 296812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pdest = ppc440spe_chan->pdest; 296912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 297012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (qdest) { 297112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_cdb *hw_desc; 297212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 297312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 297412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_first_entry(&sw_desc->group_list, 297512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 297612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 297712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 297812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 297912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (pdest) { 298012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 298112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 298212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 298312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 298412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 298512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 298612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->hw_next = list_entry(iter->chain_node.next, 298712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 298812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 298912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc = iter->hw_desc; 299012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 299112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->src_cnt = 0; 299212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->dst_cnt = 0; 299312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 0, 299412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->qdest, 0); 299512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest); 299612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 299712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin len); 299812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = 0; 299912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* override qdest to preserve original Q */ 300012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qdest = ppc440spe_chan->qdest; 300112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 300212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 300312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup destinations for P/Q ops */ 300412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest); 300512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 300612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup zero QWORDs into DCHECK CDBs */ 300712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idst = dst_cnt; 300812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_reverse(iter, &sw_desc->group_list, 300912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 301012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 301112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * The last CDB corresponds to Q-parity check, 301212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the one before last CDB corresponds 301312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * P-parity check 301412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 301512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (idst == DMA_DEST_MAX_NUM) { 301612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (idst == dst_cnt) { 301712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_QCHECK, 301812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags); 301912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 302012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_PCHECK, 302112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags); 302212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 302312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 302412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (qdest) { 302512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_QCHECK, 302612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags); 302712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 302812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_PCHECK, 302912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &iter->flags); 303012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 303112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 303212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->xor_check_result = pqres; 303312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 303412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 303512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * set it to zero, if check fail then result will 303612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * be updated 303712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 303812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *iter->xor_check_result = 0; 303912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dcheck(iter, ppc440spe_chan, 304012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_qword); 304112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 304212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(--dst_cnt)) 304312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 304412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 304512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 304612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup sources and mults for P/Q ops */ 304712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_continue_reverse(iter, &sw_desc->group_list, 304812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 304912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 305012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 mult_dst; 305112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 305212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 305312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 305412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB, 305512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src[src_cnt - 1]); 305612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (qdest) { 305712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 : 305812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1; 305912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, 306012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_MULT1_OFF, 306112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst, 306212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin scf[src_cnt - 1]); 306312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 306412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(--src_cnt)) 306512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 306612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 306712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 306812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 306912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return sw_desc ? &sw_desc->async_tx : NULL; 307012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 307112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 307212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 307312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for 307412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * XOR ZERO_SUM operation 307512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 307612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum( 307712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt, 307812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size_t len, enum sum_check_flags *result, unsigned long flags) 307912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 308012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_async_tx_descriptor *tx; 308112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t pq[2]; 308212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 308312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* validate P, disable Q */ 308412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pq[0] = src[0]; 308512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pq[1] = 0; 308612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin flags |= DMA_PREP_PQ_DISABLE_Q; 308712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 308812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1], 308912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin src_cnt - 1, 0, len, 309012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin result, flags); 309112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return tx; 309212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 309312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 309412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 309512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_set_dest - set destination address into descriptor 309612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 309712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc, 309812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index) 309912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 310012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 310112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 310212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(index >= sw_desc->dst_cnt); 310312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 310412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 310512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 310612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 310712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 310812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 310912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* to do: support transfers lengths > 311012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT 311112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 311212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(sw_desc->group_head, 311312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan, 0, addr, index); 311412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 311512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 311612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_get_group_entry(sw_desc, index); 311712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(sw_desc, 311812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan, 0, addr, index); 311912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 312012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 312112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 312212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 312312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter, 312412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, dma_addr_t addr) 312512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 312612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* To clear destinations update the descriptor 312712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (P or Q depending on index) as follows: 312812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * addr is destination (0 corresponds to SG2): 312912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 313012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0); 313112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 313212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* ... and the addr is source: */ 313312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr); 313412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 313512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* addr is always SG2 then the mult is always DST1 */ 313612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 313712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1, 1); 313812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 313912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 314012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 314112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_pq_set_dest - set destination address into descriptor 314212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for the PQXOR operation 314312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 314412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc, 314512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t *addrs, unsigned long flags) 314612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 314712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter; 314812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 314912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t paddr, qaddr; 315012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr = 0, ppath, qpath; 315112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index = 0, i; 315212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 315312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 315412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 315512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_PQ_DISABLE_P) 315612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr = 0; 315712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 315812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr = addrs[0]; 315912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 316012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (flags & DMA_PREP_PQ_DISABLE_Q) 316112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qaddr = 0; 316212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 316312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qaddr = addrs[1]; 316412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 316512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!paddr || !qaddr) 316612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr = paddr ? paddr : qaddr; 316712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 316812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 316912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 317012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 317112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* walk through the WXOR source list and set P/Q-destinations 317212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for each slot: 317312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 317412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 317512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is WXOR-only chain; may have 1/2 zero descs */ 317612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 317712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index++; 317812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 317912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index++; 318012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 318112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, index); 318212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (addr) { 318312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* one destination */ 318412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, 318512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list, chain_node) 318612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 318712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, addr, 0); 318812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 318912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* two destinations */ 319012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, 319112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list, chain_node) { 319212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 319312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, paddr, 0); 319412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 319512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, qaddr, 1); 319612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 319712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 319812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 319912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index) { 320012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* To clear destinations update the descriptor 320112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * (1st,2nd, or both depending on flags) 320212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 320312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index = 0; 320412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, 320512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) { 320612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry( 320712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc, index++); 320812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_zero_op(iter, chan, 320912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr); 321012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 321112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 321212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, 321312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) { 321412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry( 321512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc, index++); 321612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_zero_op(iter, chan, 321712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qaddr); 321812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 321912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 322012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return; 322112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 322212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 322312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* This is RXOR-only or RXOR/WXOR mixed chain */ 322412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 322512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* If we want to include destination into calculations, 322612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * then make dest addresses cued with mult=1 (XOR). 322712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 322812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ? 322912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB : 323012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE | 323112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << DMA_CUED_MULT1_OFF); 323212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ? 323312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB : 323412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE | 323512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << DMA_CUED_MULT1_OFF); 323612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 323712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup destination(s) in RXOR slot(s) */ 323812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, index++); 323912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 324012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr ? ppath : qpath, 324112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr ? paddr : qaddr, 0); 324212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addr) { 324312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* two destinations */ 324412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 324512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index++); 324612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 324712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qpath, qaddr, 0); 324812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 324912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 325012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) { 325112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup destination(s) in remaining WXOR 325212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * slots 325312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 325412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 325512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index); 325612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (addr) { 325712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* one destination */ 325812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, 325912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list, 326012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) 326112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr( 326212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter, chan, 326312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, 326412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr, 0); 326512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 326612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 326712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* two destinations */ 326812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, 326912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->group_list, 327012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 327112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr( 327212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter, chan, 327312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, 327412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr, 0); 327512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr( 327612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter, chan, 327712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, 327812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qaddr, 1); 327912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 328012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 328112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 328212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 328312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 328412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 328512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 328612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 328712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA2 descriptors have only 1 destination, so there are 328812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * two chains - one for each dest. 328912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * If we want to include destination into calculations, 329012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * then make dest addresses cued with mult=1 (XOR). 329112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 329212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ? 329312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB : 329412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE | 329512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << DMA_CUED_MULT1_OFF); 329612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 329712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ? 329812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_HB : 329912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE | 330012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << DMA_CUED_MULT1_OFF); 330112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 330212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 0); 330312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < sw_desc->descs_per_op; i++) { 330412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 330512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr ? ppath : qpath, 330612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin paddr ? paddr : qaddr, 0); 330712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 330812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 330912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 331012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 331112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 331212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addr) { 331312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Two destinations; setup Q here */ 331412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 331512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->descs_per_op); 331612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < sw_desc->descs_per_op; i++) { 331712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, 331812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan, qpath, qaddr, 0); 331912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = list_entry(iter->chain_node.next, 332012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 332112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 332212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 332312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 332412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 332512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 332612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 332712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 332812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 332912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 333012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor 333112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for the PQ_ZERO_SUM operation 333212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 333312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pqzero_sum_set_dest( 333412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, 333512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t paddr, dma_addr_t qaddr) 333612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 333712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter, *end; 333812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 333912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr = 0; 334012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int idx; 334112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 334212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 334312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 334412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* walk through the WXOR source list and set P/Q-destinations 334512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * for each slot 334612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 334712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idx = (paddr && qaddr) ? 2 : 1; 334812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set end */ 334912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_reverse(end, &sw_desc->group_list, 335012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 335112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!(--idx)) 335212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 335312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 335412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set start */ 335512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idx = (paddr && qaddr) ? 2 : 1; 335612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, idx); 335712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 335812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (paddr && qaddr) { 335912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* two destinations */ 336012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &sw_desc->group_list, 336112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 336212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(iter == end)) 336312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 336412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 336512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, paddr, 0); 336612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 336712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, qaddr, 1); 336812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 336912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 337012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* one destination */ 337112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addr = paddr ? paddr : qaddr; 337212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_from(iter, &sw_desc->group_list, 337312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 337412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(iter == end)) 337512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 337612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_dest_addr(iter, chan, 337712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_XOR_BASE, addr, 0); 337812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 337912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 338012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 338112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* The remaining descriptors are DATACHECK. These have no need in 338212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * destination. Actually, these destinations are used there 338312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * as sources for check operation. So, set addr as source. 338412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 338512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr); 338612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 338712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!addr) { 338812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin end = list_entry(end->chain_node.next, 338912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 339012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr); 339112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 339212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 339312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 339412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 339512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor 339612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 339712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic inline void ppc440spe_desc_set_xor_src_cnt( 339812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 339912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt) 340012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 340112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *hw_desc = desc->hw_desc; 340212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 340312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc &= ~XOR_CDCR_OAC_MSK; 340412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin hw_desc->cbc |= src_cnt; 340512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 340612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 340712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 340812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_pq_set_src - set source address into descriptor 340912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 341012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc, 341112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index) 341212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 341312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 341412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t haddr = 0; 341512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter = NULL; 341612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 341712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 341812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 341912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 342012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 342112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 342212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain 342312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 342412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 342512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR-only or RXOR/WXOR operation */ 342612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int iskip = test_bit(PPC440SPE_DESC_RXOR12, 342712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags) ? 2 : 3; 342812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 342912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == 0) { 343012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 1st slot (RXOR) */ 343112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* setup sources region (R1-2-3, R1-2-4, 343212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * or R1-2-5) 343312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 343412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_RXOR12, 343512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) 343612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_RXOR12 << 343712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 343812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else if (test_bit(PPC440SPE_DESC_RXOR123, 343912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) 344012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_RXOR123 << 344112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 344212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else if (test_bit(PPC440SPE_DESC_RXOR124, 344312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) 344412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_RXOR124 << 344512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 344612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else if (test_bit(PPC440SPE_DESC_RXOR125, 344712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags)) 344812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_RXOR125 << 344912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CUED_REGION_OFF; 345012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 345112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 345212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr |= DMA_CUED_XOR_BASE; 345312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 0); 345412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (index < iskip) { 345512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 1st slot (RXOR) 345612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * shall actually set source address only once 345712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * instead of first <iskip> 345812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 345912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = NULL; 346012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 346112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 2nd/3d and next slots (WXOR); 346212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * skip first slot with RXOR 346312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 346412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_CUED_XOR_HB; 346512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 346612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index - iskip + sw_desc->dst_cnt); 346712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 346812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 346912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int znum = 0; 347012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 347112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR-only operation; skip first slots with 347212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * zeroing destinations 347312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 347412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 347512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin znum++; 347612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 347712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin znum++; 347812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 347912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr = DMA_CUED_XOR_HB; 348012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 348112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index + znum); 348212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 348312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 348412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(iter)) { 348512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr); 348612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 348712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!index && 348812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) && 348912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt == 2) { 349012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* if we have two destinations for RXOR, then 349112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * setup source in the second descr too 349212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 349312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 1); 349412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(iter, chan, 0, 349512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin haddr, addr); 349612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 349712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 349812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 349912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 350012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 350112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA2 may do Biskup */ 350212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = sw_desc->group_head; 350312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (iter->dst_cnt == 2) { 350412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* both P & Q calculations required; set P src here */ 350512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_set_src(iter, index, addr); 350612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 350712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* this is for Q */ 350812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 350912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->descs_per_op); 351012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 351112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_set_src(iter, index, addr); 351212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 351312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 351412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 351512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 351612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 351712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor 351812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 351912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_memcpy_xor_set_src( 352012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, 352112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t addr, int index) 352212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 352312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 352412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 352512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 352612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = sw_desc->group_head; 352712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 352812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(sw_desc)) 352912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr); 353012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 353112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 353212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 353312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_inc_addr - 353412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 353512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_inc_addr( 353612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 353712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_rxor *cursor, int index, int src_cnt) 353812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 353912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count++; 354012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt - 1) { 354112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count); 354212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cursor->addr_count == XOR_MAX_OPS) { 354312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count); 354412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count = 0; 354512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->desc_count++; 354612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 354712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 354812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 354912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 355012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB 355112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 355212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_dma2rxor_prep_src( 355312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *hdesc, 355412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_rxor *cursor, int index, 355512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int src_cnt, u32 addr) 355612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 355712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int rval = 0; 355812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 sign; 355912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc = hdesc; 356012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 356112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 356212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < cursor->desc_count; i++) { 356312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc = list_entry(hdesc->chain_node.next, 356412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 356512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 356612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 356712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 356812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (cursor->state) { 356912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 0: 357012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (addr == cursor->addrl + cursor->len) { 357112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct RXOR */ 357212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 1; 357312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count++; 357412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 357512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 357612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 357712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR12 << DMA_CUED_REGION_OFF); 357812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 357912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 358012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 358112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (cursor->addrl == addr + cursor->len) { 358212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse RXOR */ 358312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 1; 358412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count++; 358512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(cursor->addr_count, &desc->reverse_flags[0]); 358612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 358712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 358812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 358912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR12 << DMA_CUED_REGION_OFF); 359012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 359112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 359212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 359312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 359412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "Cannot build " 359512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "DMA2 RXOR command block.\n"); 359612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG(); 359712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 359812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 359912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 1: 360012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sign = test_bit(cursor->addr_count, 360112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc->reverse_flags) 360212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ? -1 : 1; 360312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-2 || (sign == -1 360412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin && addr != cursor->addrl - 2*cursor->len)) { 360512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 0; 360612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 1; 360712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addrl = addr; 360812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 360912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 361012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR12 << DMA_CUED_REGION_OFF); 361112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 361212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 361312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (addr == cursor->addrl + 2*sign*cursor->len) { 361412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 2; 361512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 0; 361612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 361712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 361812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR123 << DMA_CUED_REGION_OFF); 361912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 362012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 362112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 362212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 362312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (addr == cursor->addrl + 3*cursor->len) { 362412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 2; 362512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 0; 362612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 362712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 362812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR124 << DMA_CUED_REGION_OFF); 362912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 363012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 363112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 363212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 363312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else if (addr == cursor->addrl + 4*cursor->len) { 363412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 2; 363512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 0; 363612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 363712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 363812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR125 << DMA_CUED_REGION_OFF); 363912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == src_cnt-1) { 364012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 364112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 364212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 364312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 364412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 0; 364512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count = 1; 364612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addrl = addr; 364712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_region(desc, 364812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addr_count, 364912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_RXOR12 << DMA_CUED_REGION_OFF); 365012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 365112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 365212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 365312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 365412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case 2: 365512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 0; 365612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->addrl = addr; 365712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->xor_count++; 365812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index) { 365912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_inc_addr( 366012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc, cursor, index, src_cnt); 366112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 366212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 366312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 366412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 366512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return rval; 366612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 366712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 366812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 366912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that 367012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call 367112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 367212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_set_src( 367312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 367412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index, dma_addr_t addr) 367512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 367612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 367712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int k = 0, op = 0, lop = 0; 367812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 367912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* get the RXOR operand which corresponds to index addr */ 368012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (op <= index) { 368112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin lop = op; 368212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (k == XOR_MAX_OPS) { 368312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin k = 0; 368412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc = list_entry(desc->chain_node.next, 368512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, chain_node); 368612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb = desc->hw_desc; 368712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 368812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 368912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) == 369012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (DMA_RXOR12 << DMA_CUED_REGION_OFF)) 369112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op += 2; 369212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 369312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op += 3; 369412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 369512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 369612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(k < 1); 369712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 369812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(k-1, desc->reverse_flags)) { 369912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse operand order; put last op in RXOR group */ 370012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == op - 1) 370112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_src(desc, k - 1, addr); 370212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 370312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct operand order; put first op in RXOR group */ 370412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index == lop) 370512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_src(desc, k - 1, addr); 370612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 370712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 370812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 370912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 371012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that 371112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call 371212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 371312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_dma2rxor_set_mult( 371412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *desc, 371512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int index, u8 mult) 371612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 371712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct xor_cb *xcb = desc->hw_desc; 371812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int k = 0, op = 0, lop = 0; 371912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 372012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* get the RXOR operand which corresponds to index mult */ 372112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin while (op <= index) { 372212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin lop = op; 372312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (k == XOR_MAX_OPS) { 372412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin k = 0; 372512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin desc = list_entry(desc->chain_node.next, 372612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot, 372712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node); 372812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin xcb = desc->hw_desc; 372912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 373012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 373112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) == 373212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (DMA_RXOR12 << DMA_CUED_REGION_OFF)) 373312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op += 2; 373412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 373512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin op += 3; 373612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 373712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 373812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(k < 1); 373912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(k-1, desc->reverse_flags)) { 374012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* reverse order */ 374112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult); 374212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 374312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* direct order */ 374412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult); 374512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 374612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 374712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 374812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 374912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_init_rxor_cursor - 375012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 375112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor) 375212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 375312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(cursor, 0, sizeof(struct ppc440spe_rxor)); 375412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin cursor->state = 2; 375512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 375612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 375712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 375812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into 375912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * descriptor for the PQXOR operation 376012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 376112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_pq_set_src_mult( 376212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, 376312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned char mult, int index, int dst_pos) 376412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 376512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 376612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 mult_idx, mult_dst; 376712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL; 376812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 376912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 377012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 377112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (chan->device->id) { 377212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 377312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 377412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 377512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int region = test_bit(PPC440SPE_DESC_RXOR12, 377612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &sw_desc->flags) ? 2 : 3; 377712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 377812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (index < region) { 377912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RXOR multipliers */ 378012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 378112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt - 1); 378212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc->dst_cnt == 2) 378312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter1 = ppc440spe_get_group_entry( 378412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc, 0); 378512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 378612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx = DMA_CUED_MULT1_OFF + (index << 3); 378712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst = DMA_CDB_SG_SRC; 378812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 378912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR multiplier */ 379012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 379112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin index - region + 379212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->dst_cnt); 379312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx = DMA_CUED_MULT1_OFF; 379412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst = dst_pos ? DMA_CDB_SG_DST2 : 379512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA_CDB_SG_DST1; 379612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 379712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 379812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int znum = 0; 379912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 380012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* WXOR-only; 380112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * skip first slots with destinations (if ZERO_DST has 380212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * place) 380312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 380412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 380512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin znum++; 380612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 380712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin znum++; 380812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 380912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, index + znum); 381012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx = DMA_CUED_MULT1_OFF; 381112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1; 381212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 381312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 381412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (likely(iter)) { 381512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter, chan, 381612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx, mult_dst, mult); 381712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 381812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (unlikely(iter1)) { 381912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* if we have two destinations for RXOR, then 382012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * we've just set Q mult. Set-up P now. 382112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 382212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_src_mult(iter1, chan, 382312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mult_idx, mult_dst, 1); 382412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 382512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 382612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 382712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 382812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 382912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 383012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = sw_desc->group_head; 383112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc->dst_cnt == 2) { 383212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* both P & Q calculations required; set P mult here */ 383312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_set_mult(iter, index, 1); 383412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 383512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* and then set Q mult */ 383612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter = ppc440spe_get_group_entry(sw_desc, 383712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->descs_per_op); 383812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 383912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_dma2rxor_set_mult(iter, index, mult); 384012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 384112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 384212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 384312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 384412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 384512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_free_chan_resources - free the resources allocated 384612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 384712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_free_chan_resources(struct dma_chan *chan) 384812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 384912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 385012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *iter, *_iter; 385112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int in_use_descs = 0; 385212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 385312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 385412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_slot_cleanup(ppc440spe_chan); 385512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 385612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&ppc440spe_chan->lock); 385712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain, 385812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chain_node) { 385912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin in_use_descs++; 386012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&iter->chain_node); 386112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 386212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe_reverse(iter, _iter, 386312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &ppc440spe_chan->all_slots, slot_node) { 386412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&iter->slot_node); 386512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(iter); 386612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->slots_allocated--; 386712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 386812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->last_used = NULL; 386912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 387012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 387112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d %s slots_allocated %d\n", 387212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->device->id, 387312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, ppc440spe_chan->slots_allocated); 387412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&ppc440spe_chan->lock); 387512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 387612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* one is ok since we left it on there on purpose */ 387712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (in_use_descs > 1) 387812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n", 387912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin in_use_descs - 1); 388012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 388112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 388212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 38830793448187643b50af89d36b08470baf45a3cab4Linus Walleij * ppc440spe_adma_tx_status - poll the status of an ADMA transaction 388412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @chan: ADMA channel handle 388512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * @cookie: ADMA transaction identifier 38860793448187643b50af89d36b08470baf45a3cab4Linus Walleij * @txstate: a holder for the current state of the channel 388712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 38880793448187643b50af89d36b08470baf45a3cab4Linus Walleijstatic enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan, 38890793448187643b50af89d36b08470baf45a3cab4Linus Walleij dma_cookie_t cookie, struct dma_tx_state *txstate) 389012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 389112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 389212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin enum dma_status ret; 389312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 389412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 389596a2af41c78b1fbb1f567a3486bdc63f7b31c5fdRussell King - ARM Linux ret = dma_cookie_status(chan, cookie, txstate); 389612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret == DMA_SUCCESS) 389712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 389812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 389912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_slot_cleanup(ppc440spe_chan); 390012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 390196a2af41c78b1fbb1f567a3486bdc63f7b31c5fdRussell King - ARM Linux return dma_cookie_status(chan, cookie, txstate); 390212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 390312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 390412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 390512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_eot_handler - end of transfer interrupt handler 390612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 390712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data) 390812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 390912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan = data; 391012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 391112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, 391212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s\n", chan->device->id, __func__); 391312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 391412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_schedule(&chan->irq_tasklet); 391512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_device_clear_eot_status(chan); 391612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 391712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return IRQ_HANDLED; 391812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 391912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 392012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 392112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_err_handler - DMA error interrupt handler; 392212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * do the same things as a eot handler 392312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 392412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic irqreturn_t ppc440spe_adma_err_handler(int irq, void *data) 392512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 392612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan = data; 392712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 392812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, 392912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s\n", chan->device->id, __func__); 393012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 393112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_schedule(&chan->irq_tasklet); 393212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_device_clear_eot_status(chan); 393312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 393412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return IRQ_HANDLED; 393512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 393612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 393712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 393812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_test_callback - called when test operation has been done 393912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 394012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_test_callback(void *unused) 394112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 394212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin complete(&ppc440spe_r6_test_comp); 394312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 394412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 394512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 394612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w 394712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 394812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_issue_pending(struct dma_chan *chan) 394912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 395012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 395112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 395212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 395312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(ppc440spe_chan->device->common.dev, 395412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id, 395512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__, ppc440spe_chan->pending); 395612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 395712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_chan->pending) { 395812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan->pending = 0; 395912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_append(ppc440spe_chan); 396012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 396112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 396212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 396312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 396412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines 396512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * use FIFOs (as opposite to chains used in XOR) so this is a XOR 396612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * specific operation) 396712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 396812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan) 396912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 397012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 397112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cookie_t cookie; 397212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int slot_cnt, slots_per_op; 397312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 397412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(chan->device->common.dev, 397512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "ppc440spe adma%d: %s\n", chan->device->id, __func__); 397612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 397712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&chan->lock); 397812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op); 397912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op); 398012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 398112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin group_start = sw_desc->group_head; 398212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_splice_init(&sw_desc->group_list, &chan->chain); 398312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin async_tx_ack(&sw_desc->async_tx); 398412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_null_xor(group_start); 398512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 39862a926e46022ad7a03e0ac167d8c2b0d88c12c5a8Russell King - ARM Linux cookie = dma_cookie_assign(&sw_desc->async_tx); 398712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 398812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* initialize the completed cookie to be less than 398912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the most recently used cookie 399012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 39914d4e58de32a192fea65ab84509d17d199bd291c8Russell King - ARM Linux chan->common.completed_cookie = cookie - 1; 399212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 399312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* channel should not be busy */ 399412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin BUG_ON(ppc440spe_chan_is_busy(chan)); 399512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 399612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set the descriptor address */ 399712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc); 399812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 399912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* run the descriptor */ 400012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan_run(chan); 400112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 400212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin printk(KERN_ERR "ppc440spe adma%d" 400312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " failed to allocate null descriptor\n", 400412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device->id); 400512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 400612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 400712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 400812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 400912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully. 401012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * For this we just perform one WXOR operation with the same source 401112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * and destination addresses, the GF-multiplier is 1; so if RAID-6 401212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * capabilities are enabled then we'll get src/dst filled with zero. 401312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 401412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan) 401512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 401612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_desc_slot *sw_desc, *iter; 401712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct page *pg; 401812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin char *a; 401912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr_t dma_addr, addrs[2]; 402012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long op = 0; 402112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int rval = 0; 402212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 402312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin set_bit(PPC440SPE_DESC_WXOR, &op); 402412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 402512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pg = alloc_page(GFP_KERNEL); 402612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!pg) 402712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENOMEM; 402812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 402912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_bh(&chan->lock); 403012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1); 403112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (sw_desc) { 403212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 1 src, 1 dsr, int_ena, WXOR */ 403312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op); 403412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 403512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE); 403612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iter->unmap_len = PAGE_SIZE; 403712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 403812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 403912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rval = -EFAULT; 404012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 404112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto exit; 404212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 404312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_unlock_bh(&chan->lock); 404412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 404512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Fill the test page with ones */ 404612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin memset(page_address(pg), 0xFF, PAGE_SIZE); 404712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_addr = dma_map_page(chan->device->dev, pg, 0, 404812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 404912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 405012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup addresses */ 405112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0); 405212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0); 405312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addrs[0] = dma_addr; 405412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin addrs[1] = 0; 405512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q); 405612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 405712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin async_tx_ack(&sw_desc->async_tx); 405812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.callback = ppc440spe_test_callback; 405912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sw_desc->async_tx.callback_param = NULL; 406012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 406112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin init_completion(&ppc440spe_r6_test_comp); 406212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 406312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_tx_submit(&sw_desc->async_tx); 406412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_issue_pending(&chan->common); 406512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 406612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin wait_for_completion(&ppc440spe_r6_test_comp); 406712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 406812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Now check if the test page is zeroed */ 406912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin a = page_address(pg); 407012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) { 407112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* page is zero - RAID-6 enabled */ 407212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rval = 0; 407312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 407412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RAID-6 was not enabled */ 407512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin rval = -EINVAL; 407612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 407712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinexit: 407812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(pg); 407912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return rval; 408012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 408112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 408212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev) 408312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 408412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (adev->id) { 408512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 408612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 408712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_MEMCPY, adev->common.cap_mask); 408812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask); 408912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_PQ, adev->common.cap_mask); 409012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask); 409112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask); 409212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 409312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 409412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_XOR, adev->common.cap_mask); 409512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_PQ, adev->common.cap_mask); 409612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask); 409712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.cap_mask = adev->common.cap_mask; 409812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 409912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 410012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 410112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set base routines */ 410212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_alloc_chan_resources = 410312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_alloc_chan_resources; 410412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_free_chan_resources = 410512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_free_chan_resources; 41060793448187643b50af89d36b08470baf45a3cab4Linus Walleij adev->common.device_tx_status = ppc440spe_adma_tx_status; 410712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_issue_pending = ppc440spe_adma_issue_pending; 410812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 410912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set prep routines based on capability */ 411012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) { 411112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_memcpy = 411212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_memcpy; 411312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 411412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) { 411512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_xor = XOR_MAX_OPS; 411612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_xor = 411712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_xor; 411812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 411912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) { 412012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (adev->id) { 412112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 412212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_set_maxpq(&adev->common, 412312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0); 412412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 412512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 412612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_set_maxpq(&adev->common, 412712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0); 412812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 412912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_XOR_ID: 413012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_pq = XOR_MAX_OPS * 3; 413112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 413212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 413312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_pq = 413412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_pq; 413512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 413612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) { 413712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (adev->id) { 413812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 413912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_pq = DMA0_FIFO_SIZE / 414012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sizeof(struct dma_cdb); 414112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 414212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 414312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_pq = DMA1_FIFO_SIZE / 414412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sizeof(struct dma_cdb); 414512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 414612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 414712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_pq_val = 414812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_pqzero_sum; 414912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 415012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) { 415112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin switch (adev->id) { 415212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA0_ID: 415312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_xor = DMA0_FIFO_SIZE / 415412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sizeof(struct dma_cdb); 415512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 415612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin case PPC440SPE_DMA1_ID: 415712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.max_xor = DMA1_FIFO_SIZE / 415812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sizeof(struct dma_cdb); 415912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin break; 416012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 416112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_xor_val = 416212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_xor_zero_sum; 416312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 416412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) { 416512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.device_prep_dma_interrupt = 416612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_prep_dma_interrupt; 416712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 416812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: " 416912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "( %s%s%s%s%s%s%s)\n", 417012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_name(adev->dev), 417112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "", 417212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "", 417312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "", 417412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "", 417512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "", 417612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : ""); 417712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 417812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 417912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev, 418012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan, 418112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int *initcode) 418212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 41832dc11581376829303b98eadb2de253bee065a56aGrant Likely struct platform_device *ofdev; 418412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct device_node *np; 418512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int ret; 418612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 41872dc11581376829303b98eadb2de253bee065a56aGrant Likely ofdev = container_of(adev->dev, struct platform_device, dev); 41883e6b02d9f5a9715f7d4ff9e0978e5f9cef53d31fDan Williams np = ofdev->dev.of_node; 418912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id != PPC440SPE_XOR_ID) { 419012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->err_irq = irq_of_parse_and_map(np, 1); 419112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->err_irq == NO_IRQ) { 419212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_warn(adev->dev, "no err irq resource?\n"); 419312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *initcode = PPC_ADMA_INIT_IRQ2; 419412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->err_irq = -ENXIO; 419512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else 419612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin atomic_inc(&ppc440spe_adma_err_irq_ref); 419712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 419812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->err_irq = -ENXIO; 419912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 420012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 420112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->irq = irq_of_parse_and_map(np, 0); 420212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->irq == NO_IRQ) { 420312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(adev->dev, "no irq resource\n"); 420412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *initcode = PPC_ADMA_INIT_IRQ1; 420512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENXIO; 420612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_irq_map; 420712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 420812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_dbg(adev->dev, "irq %d, err irq %d\n", 420912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->irq, adev->err_irq); 421012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 421112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = request_irq(adev->irq, ppc440spe_adma_eot_handler, 421212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 0, dev_driver_string(adev->dev), chan); 421312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) { 421412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(adev->dev, "can't request irq %d\n", 421512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->irq); 421612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *initcode = PPC_ADMA_INIT_IRQ1; 421712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -EIO; 421812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_req1; 421912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 422012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 422112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* only DMA engines have a separate error IRQ 422212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * so it's Ok if err_irq < 0 in XOR engine case. 422312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 422412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->err_irq > 0) { 422512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* both DMA engines share common error IRQ */ 422612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = request_irq(adev->err_irq, 422712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_err_handler, 422812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin IRQF_SHARED, 422912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_driver_string(adev->dev), 423012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan); 423112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) { 423212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(adev->dev, "can't request irq %d\n", 423312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->err_irq); 423412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin *initcode = PPC_ADMA_INIT_IRQ2; 423512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -EIO; 423612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_req2; 423712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 423812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 423912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 424012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) { 424112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* enable XOR engine interrupts */ 424212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT | 424312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT, 424412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &adev->xor_reg->ier); 424512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 424612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 mask, enable; 424712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 424812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe"); 424912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!np) { 425012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't find I2O device tree node\n", 425112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 425212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 425312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_req2; 425412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 425512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->i2o_reg = of_iomap(np, 0); 425612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!adev->i2o_reg) { 425712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to map I2O registers\n", __func__); 425812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 425912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -EINVAL; 426012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_req2; 426112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 426212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 426312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Unmask 'CS FIFO Attention' interrupts and 426412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * enable generating interrupts on errors 426512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 426612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin enable = (adev->id == PPC440SPE_DMA0_ID) ? 426712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) : 426812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM); 426912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mask = ioread32(&adev->i2o_reg->iopim) & enable; 427012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(mask, &adev->i2o_reg->iopim); 427112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 427212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 427312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 427412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_req2: 427512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin free_irq(adev->irq, chan); 427612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_req1: 427712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin irq_dispose_mapping(adev->irq); 427812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_irq_map: 427912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->err_irq > 0) { 428012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) 428112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin irq_dispose_mapping(adev->err_irq); 428212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 428312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 428412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 428512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 428612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev, 428712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan) 428812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 428912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 mask, disable; 429012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 429112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) { 429212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* disable XOR engine interrupts */ 429312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mask = ioread32be(&adev->xor_reg->ier); 429412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT | 429512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT); 429612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(mask, &adev->xor_reg->ier); 429712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 429812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* disable DMAx engine interrupts */ 429912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin disable = (adev->id == PPC440SPE_DMA0_ID) ? 430012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) : 430112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM); 430212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mask = ioread32(&adev->i2o_reg->iopim) | disable; 430312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(mask, &adev->i2o_reg->iopim); 430412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 430512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin free_irq(adev->irq, chan); 430612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin irq_dispose_mapping(adev->irq); 430712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->err_irq > 0) { 430812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin free_irq(adev->err_irq, chan); 430912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) { 431012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin irq_dispose_mapping(adev->err_irq); 431112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->i2o_reg); 431212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 431312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 431412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 431512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 431612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 431712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_probe - probe the asynch device 431812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 4319463a1f8b3ceebe990ca9a8c7cc2e51ee42cb48ebBill Pembertonstatic int ppc440spe_adma_probe(struct platform_device *ofdev) 432012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 432105c02542c20aa00dc9a66f4bfb1a89d1131457f2Anatolij Gustschin struct device_node *np = ofdev->dev.of_node; 432212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct resource res; 432312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_device *adev; 432412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *chan; 432512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc_dma_chan_ref *ref, *_ref; 432612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int ret = 0, initcode = PPC_ADMA_INIT_OK; 432712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const u32 *idx; 432812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int len; 432912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin void *regs; 433012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 id, pool_size; 433112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 433212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (of_device_is_compatible(np, "amcc,xor-accelerator")) { 433312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin id = PPC440SPE_XOR_ID; 433412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* As far as the XOR engine is concerned, it does not 433512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * use FIFOs but uses linked list. So there is no dependency 433612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * between pool size to allocate and the engine configuration. 433712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 433812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pool_size = PAGE_SIZE << 1; 433912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 434012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* it is DMA0 or DMA1 */ 434112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin idx = of_get_property(np, "cell-index", &len); 434212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!idx || (len != sizeof(u32))) { 434312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "Device node %s has missing " 434412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "or invalid cell-index property\n", 434512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np->full_name); 434612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 434712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 434812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin id = *idx; 434912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMA0,1 engines use FIFO to maintain CDBs, so we 435012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * should allocate the pool accordingly to size of this 435112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * FIFO. Thus, the pool size depends on the FIFO depth: 435212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * how much CDBs pointers the FIFO may contain then so 435312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * much CDBs we should provide in the pool. 435412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * That is 435512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * CDB size = 32B; 435612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * CDBs number = (DMA0_FIFO_SIZE >> 3); 435712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Pool size = CDBs number * CDB size = 435812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2. 435912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 436012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pool_size = (id == PPC440SPE_DMA0_ID) ? 436112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA0_FIFO_SIZE : DMA1_FIFO_SIZE; 436212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pool_size <<= 2; 436312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 436412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 436512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (of_address_to_resource(np, 0, &res)) { 436612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to get memory resource\n"); 436712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_MEMRES; 436812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 436912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out; 437012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 437112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 437212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!request_mem_region(res.start, resource_size(&res), 437312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_driver_string(&ofdev->dev))) { 4374a584bff5efae8c1d026e3a930e3d13a90264fafcJoe Perches dev_err(&ofdev->dev, "failed to request memory region %pR\n", 4375a584bff5efae8c1d026e3a930e3d13a90264fafcJoe Perches &res); 437612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_MEMREG; 437712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -EBUSY; 437812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out; 437912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 438012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 438112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* create a device */ 438212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev = kzalloc(sizeof(*adev), GFP_KERNEL); 438312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!adev) { 438412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to allocate device\n"); 438512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_ALLOC; 438612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 438712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_adev_alloc; 438812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 438912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 439012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->id = id; 439112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->pool_size = pool_size; 439212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* allocate coherent memory for hardware descriptors */ 439312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev, 439412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->pool_size, &adev->dma_desc_pool, 439512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin GFP_KERNEL); 439612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->dma_desc_pool_virt == NULL) { 439712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent " 439812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "memory for hardware descriptors\n", 439912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->pool_size); 440012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_COHERENT; 440112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 440212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_dma_alloc; 440312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 4404d73111c6d43f08d697a3d986fe3bd8ca796a2096Masanari Iida dev_dbg(&ofdev->dev, "allocated descriptor pool virt 0x%p phys 0x%llx\n", 440512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool); 440612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 440712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin regs = ioremap(res.start, resource_size(&res)); 440812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!regs) { 440912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to ioremap regs!\n"); 441012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_regs_alloc; 441112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 441212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 441312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) { 441412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->xor_reg = regs; 441512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Reset XOR */ 441612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr); 441712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr); 441812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 441912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ? 442012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin DMA0_FIFO_SIZE : DMA1_FIFO_SIZE; 442112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_reg = regs; 442212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* DMAx_FIFO_SIZE is defined in bytes, 442312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * <fsiz> - is defined in number of CDB pointers (8byte). 442412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * DMA FIFO Length = CSlength + CPlength, where 442512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * CSlength = CPlength = (fsiz + 1) * 8. 442612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 442712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2), 442812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &adev->dma_reg->fsiz); 442912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Configure DMA engine */ 443012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN, 443112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &adev->dma_reg->cfg); 443212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Clear Status */ 443312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(~0, &adev->dma_reg->dsts); 443412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 443512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 443612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dev = &ofdev->dev; 443712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->common.dev = &ofdev->dev; 443812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&adev->common.channels); 4439dd3daca162f7411448dd80a8872a002c43cfd8e5Jingoo Han platform_set_drvdata(ofdev, adev); 444012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 444112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* create a channel */ 444212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan = kzalloc(sizeof(*chan), GFP_KERNEL); 444312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!chan) { 444412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "can't allocate channel structure\n"); 444512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_CHANNEL; 444612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 444712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_chan_alloc; 444812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 444912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 445012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin spin_lock_init(&chan->lock); 445112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&chan->chain); 445212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&chan->all_slots); 445312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->device = adev; 445412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->common.device = &adev->common; 44558ac695463f37af902e953d575d3f782e32e170daRussell King - ARM Linux dma_cookie_init(&chan->common); 445612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_add_tail(&chan->common.device_node, &adev->common.channels); 445712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_init(&chan->irq_tasklet, ppc440spe_adma_tasklet, 445812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (unsigned long)chan); 445912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 446012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* allocate and map helper pages for async validation or 446112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * async_mult/async_sum_product operations on DMA0/1. 446212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 446312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id != PPC440SPE_XOR_ID) { 446412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->pdest_page = alloc_page(GFP_KERNEL); 446512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->qdest_page = alloc_page(GFP_KERNEL); 446612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!chan->pdest_page || 446712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin !chan->qdest_page) { 446812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan->pdest_page) 446912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(chan->pdest_page); 447012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan->qdest_page) 447112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(chan->qdest_page); 447212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 447312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_page_alloc; 447412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 447512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0, 447612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 447712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0, 447812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 447912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 448012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 448112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ref = kmalloc(sizeof(*ref), GFP_KERNEL); 448212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ref) { 448312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ref->chan = &chan->common; 448412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin INIT_LIST_HEAD(&ref->node); 448512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_add_tail(&ref->node, &ppc440spe_adma_chan_list); 448612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 448712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to allocate channel reference!\n"); 448812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENOMEM; 448912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_ref_alloc; 449012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 449112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 449212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode); 449312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) 449412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_irq; 449512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 449612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_init_capabilities(adev); 449712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 449812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = dma_async_device_register(&adev->common); 449912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) { 450012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin initcode = PPC_ADMA_INIT_REGISTER; 450112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dev_err(&ofdev->dev, "failed to register dma device\n"); 450212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto err_dev_reg; 450312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 450412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 450512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out; 450612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 450712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_dev_reg: 450812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_release_irqs(adev, chan); 450912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_irq: 451012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) { 451112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (chan == to_ppc440spe_adma_chan(ref->chan)) { 451212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&ref->node); 451312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ref); 451412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 451512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 451612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_ref_alloc: 451712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id != PPC440SPE_XOR_ID) { 451812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(&ofdev->dev, chan->pdest, 451912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 452012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(&ofdev->dev, chan->qdest, 452112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 452212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(chan->pdest_page); 452312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(chan->qdest_page); 452412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 452512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_page_alloc: 452612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(chan); 452712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_chan_alloc: 452812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) 452912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->xor_reg); 453012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 453112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->dma_reg); 453212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_regs_alloc: 453312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_free_coherent(adev->dev, adev->pool_size, 453412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool_virt, 453512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool); 453612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_dma_alloc: 453712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(adev); 453812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinerr_adev_alloc: 453912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin release_mem_region(res.start, resource_size(&res)); 454012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout: 454112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (id < PPC440SPE_ADMA_ENGINES_NUM) 454212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_devices[id] = initcode; 454312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 454412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 454512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 454612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 454712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/** 454812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_adma_remove - remove the asynch device 454912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 45504bf27b8b333bcd291664fd0f7d129099d474a23bGreg Kroah-Hartmanstatic int ppc440spe_adma_remove(struct platform_device *ofdev) 455112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 4552dd3daca162f7411448dd80a8872a002c43cfd8e5Jingoo Han struct ppc440spe_adma_device *adev = platform_get_drvdata(ofdev); 455305c02542c20aa00dc9a66f4bfb1a89d1131457f2Anatolij Gustschin struct device_node *np = ofdev->dev.of_node; 455412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct resource res; 455512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct dma_chan *chan, *_chan; 455612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc_dma_chan_ref *ref, *_ref; 455712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct ppc440spe_adma_chan *ppc440spe_chan; 455812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 455912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id < PPC440SPE_ADMA_ENGINES_NUM) 456012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_devices[adev->id] = -1; 456112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 456212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_async_device_unregister(&adev->common); 456312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 456412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(chan, _chan, &adev->common.channels, 456512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin device_node) { 456612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_chan = to_ppc440spe_adma_chan(chan); 456712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_release_irqs(adev, ppc440spe_chan); 456812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin tasklet_kill(&ppc440spe_chan->irq_tasklet); 456912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id != PPC440SPE_XOR_ID) { 457012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest, 457112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 457212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest, 457312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin PAGE_SIZE, DMA_BIDIRECTIONAL); 457412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(ppc440spe_chan->pdest_page); 457512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __free_page(ppc440spe_chan->qdest_page); 457612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 457712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, 457812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin node) { 457912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_chan == 458012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin to_ppc440spe_adma_chan(ref->chan)) { 458112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&ref->node); 458212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ref); 458312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 458412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 458512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin list_del(&chan->device_node); 458612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ppc440spe_chan); 458712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 458812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 458912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dma_free_coherent(adev->dev, adev->pool_size, 459012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin adev->dma_desc_pool_virt, adev->dma_desc_pool); 459112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (adev->id == PPC440SPE_XOR_ID) 459212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->xor_reg); 459312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin else 459412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(adev->dma_reg); 459512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_address_to_resource(np, 0, &res); 459612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin release_mem_region(res.start, resource_size(&res)); 459712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(adev); 459812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 459912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 460012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 460112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* 460212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * /sys driver interface to enable h/w RAID-6 capabilities 460312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/ 460412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * directory are "devices", "enable" and "poly". 460512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * "devices" shows available engines. 460612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * "enable" is used to enable RAID-6 capabilities or to check 460712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * whether these has been activated. 460812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * "poly" allows setting/checking used polynomial (for PPC440SPe only). 460912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 461012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 461112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t show_ppc440spe_devices(struct device_driver *dev, char *buf) 461212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 461312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ssize_t size = 0; 461412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i; 461512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 461612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) { 461712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_adma_devices[i] == -1) 461812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin continue; 461912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size += snprintf(buf + size, PAGE_SIZE - size, 462012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "PPC440SP(E)-ADMA.%d: %s\n", i, 462112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc_adma_errors[ppc440spe_adma_devices[i]]); 462212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 462312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return size; 462412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 462512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 462612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t show_ppc440spe_r6enable(struct device_driver *dev, char *buf) 462712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 462812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return snprintf(buf, PAGE_SIZE, 462912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "PPC440SP(e) RAID-6 capabilities are %sABLED.\n", 463012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_r6_enabled ? "EN" : "DIS"); 463112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 463212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 463312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t store_ppc440spe_r6enable(struct device_driver *dev, 463412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const char *buf, size_t count) 463512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 463612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long val; 463712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 463812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!count || count > 11) 463912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 464012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 464112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!ppc440spe_r6_tchan) 464212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EFAULT; 464312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 464412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Write a key */ 464512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sscanf(buf, "%lx", &val); 464612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val); 464712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin isync(); 464812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 464912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Verify whether it really works now */ 465012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) { 465112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_info("PPC440SP(e) RAID-6 has been activated " 465212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "successfully\n"); 465312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_r6_enabled = 1; 465412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } else { 465512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_info("PPC440SP(e) RAID-6 hasn't been activated!" 465612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin " Error key ?\n"); 465712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_r6_enabled = 0; 465812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 465912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return count; 466012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 466112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 466212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t show_ppc440spe_r6poly(struct device_driver *dev, char *buf) 466312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 466412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ssize_t size = 0; 466512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin u32 reg; 466612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 466712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#ifdef CONFIG_440SP 466812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 440SP has fixed polynomial */ 466912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg = 0x4d; 467012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#else 467112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL); 467212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg >>= MQ0_CFBHL_POLY; 467312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg &= 0xFF; 467412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#endif 467512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 467612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver " 467712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin "uses 0x1%02x polynomial.\n", reg); 467812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return size; 467912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 468012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 468112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic ssize_t store_ppc440spe_r6poly(struct device_driver *dev, 468212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin const char *buf, size_t count) 468312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 468412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned long reg, val; 468512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 468612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#ifdef CONFIG_440SP 468712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 440SP uses default 0x14D polynomial only */ 468812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 468912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin#endif 469012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 469112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!count || count > 6) 469212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 469312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 469412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* e.g., 0x14D or 0x11D */ 469512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin sscanf(buf, "%lx", &val); 469612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 469712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (val & ~0x1FF) 469812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 469912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 470012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin val &= 0xFF; 470112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL); 470212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg &= ~(0xFF << MQ0_CFBHL_POLY); 470312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin reg |= val << MQ0_CFBHL_POLY; 470412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg); 470512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 470612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return count; 470712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 470812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 470912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic DRIVER_ATTR(devices, S_IRUGO, show_ppc440spe_devices, NULL); 471012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic DRIVER_ATTR(enable, S_IRUGO | S_IWUSR, show_ppc440spe_r6enable, 471112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin store_ppc440spe_r6enable); 471212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic DRIVER_ATTR(poly, S_IRUGO | S_IWUSR, show_ppc440spe_r6poly, 471312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin store_ppc440spe_r6poly); 471412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 471512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin/* 471612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Common initialisation for RAID engines; allocate memory for 471712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * DMAx FIFOs, perform configuration common for all DMA engines. 471812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Further DMA engine specific configuration is done at probe time. 471912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 472012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic int ppc440spe_configure_raid_devices(void) 472112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 472212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct device_node *np; 472312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct resource i2o_res; 472412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin struct i2o_regs __iomem *i2o_reg; 472512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_host_t i2o_dcr_host; 472612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin unsigned int dcr_base, dcr_len; 472712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int i, ret; 472812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 472912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe"); 473012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!np) { 473112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't find I2O device tree node\n", 473212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 473312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENODEV; 473412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 473512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 473612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (of_address_to_resource(np, 0, &i2o_res)) { 473712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 473812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 473912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 474012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 474112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i2o_reg = of_iomap(np, 0); 474212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!i2o_reg) { 474312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to map I2O registers\n", __func__); 474412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 474512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -EINVAL; 474612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 474712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 474812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Get I2O DCRs base */ 474912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_base = dcr_resource_start(np, 0); 475012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_len = dcr_resource_len(np, 0); 475112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!dcr_base && !dcr_len) { 475212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't get DCR registers base/len!\n", 475312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np->full_name); 475412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 475512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(i2o_reg); 475612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENODEV; 475712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 475812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 475912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin i2o_dcr_host = dcr_map(np, dcr_base, dcr_len); 476012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!DCR_MAP_OK(i2o_dcr_host)) { 476112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to map DCRs!\n", np->full_name); 476212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 476312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(i2o_reg); 476412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENODEV; 476512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 476612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 476712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 476812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share 476912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * the base address of FIFO memory space. 477012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Actually we need twice more physical memory than programmed in the 477112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * <fsiz> register (because there are two FIFOs for each DMA: CP and CS) 477212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 477312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1, 477412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin GFP_KERNEL); 477512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!ppc440spe_dma_fifo_buf) { 477612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__); 477712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(i2o_reg); 477812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_unmap(i2o_dcr_host, dcr_len); 477912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return -ENOMEM; 478012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 478112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 478212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* 478312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Configure h/w 478412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 478512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Reset I2O/DMA */ 478612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA); 478712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin mtdcri(SDR0, DCRN_SDR0_SRST, 0); 478812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 478912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup the base address of mmaped registers */ 479012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32)); 479112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) | 479212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin I2O_REG_ENABLE); 479312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_unmap(i2o_dcr_host, dcr_len); 479412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 479512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Setup FIFO memory space base address */ 479612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(0, &i2o_reg->ifbah); 479712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal); 479812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 479912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* set zero FIFO size for I2O, so the whole 480012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * ppc440spe_dma_fifo_buf is used by DMAs. 480112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * DMAx_FIFOs will be configured while probe. 480212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 480312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iowrite32(0, &i2o_reg->ifsiz); 480412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin iounmap(i2o_reg); 480512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 480612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* To prepare WXOR/RXOR functionality we need access to 480712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * Memory Queue Module DCRs (finally it will be enabled 480812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * via /sys interface of the ppc440spe ADMA driver). 480912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 481012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe"); 481112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!np) { 481212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't find MQ device tree node\n", 481312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 481412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 481512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_free; 481612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 481712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 481812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Get MQ DCRs base */ 481912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_base = dcr_resource_start(np, 0); 482012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_len = dcr_resource_len(np, 0); 482112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!dcr_base && !dcr_len) { 482212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: can't get DCR registers base/len!\n", 482312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin np->full_name); 482412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 482512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_mq; 482612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 482712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 482812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len); 482912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) { 483012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to map DCRs!\n", np->full_name); 483112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = -ENODEV; 483212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_mq; 483312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 483412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 483512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_mq_dcr_len = dcr_len; 483612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 483712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set HB alias */ 483812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB); 483912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 484012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Set: 484112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - LL transaction passing limit to 1; 484212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - Memory controller cycle limit to 1; 484312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin * - Galois Polynomial to 0x14d (default) 484412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin */ 484512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, 484612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) | 484712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY)); 484812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 484912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin atomic_set(&ppc440spe_adma_err_irq_ref, 0); 485012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) 485112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ppc440spe_adma_devices[i] = -1; 485212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 485312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return 0; 485412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 485512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_mq: 485612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin of_node_put(np); 485712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_free: 485812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ppc440spe_dma_fifo_buf); 485912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 486012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 486112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 48624bf27b8b333bcd291664fd0f7d129099d474a23bGreg Kroah-Hartmanstatic const struct of_device_id ppc440spe_adma_of_match[] = { 486312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin { .compatible = "ibm,dma-440spe", }, 486412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin { .compatible = "amcc,xor-accelerator", }, 486512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin {}, 486612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 486712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinMODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match); 486812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 4869000061245a6797d542854106463b6b20fbdcb12eGrant Likelystatic struct platform_driver ppc440spe_adma_driver = { 487012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin .probe = ppc440spe_adma_probe, 4871a7d6e3ec28bba30d1409d70de1958edc6d9109d7Bill Pemberton .remove = ppc440spe_adma_remove, 487212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin .driver = { 487312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin .name = "PPC440SP(E)-ADMA", 487412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin .owner = THIS_MODULE, 48754018294b53d1dae026880e45f174c1cc63b5d435Grant Likely .of_match_table = ppc440spe_adma_of_match, 487612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin }, 487712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin}; 487812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 487912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic __init int ppc440spe_adma_init(void) 488012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 488112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin int ret; 488212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 488312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = ppc440spe_configure_raid_devices(); 488412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) 488512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 488612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 4887000061245a6797d542854106463b6b20fbdcb12eGrant Likely ret = platform_driver_register(&ppc440spe_adma_driver); 488812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) { 488912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to register platform driver\n", 489012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 489112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_reg; 489212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin } 489312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 489412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* Initialization status */ 489512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = driver_create_file(&ppc440spe_adma_driver.driver, 489612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_devices); 489712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) 489812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_dev; 489912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 490012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* RAID-6 h/w enable entry */ 490112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = driver_create_file(&ppc440spe_adma_driver.driver, 490212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_enable); 490312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (ret) 490412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin goto out_en; 490512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 490612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* GF polynomial to use */ 490712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin ret = driver_create_file(&ppc440spe_adma_driver.driver, 490812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_poly); 490912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin if (!ret) 491012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 491112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 491212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 491312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_enable); 491412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_en: 491512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 491612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_devices); 491712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_dev: 491812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin /* User will not be able to enable h/w RAID-6 */ 491912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin pr_err("%s: failed to create RAID-6 driver interface\n", 492012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin __func__); 4921000061245a6797d542854106463b6b20fbdcb12eGrant Likely platform_driver_unregister(&ppc440spe_adma_driver); 492212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinout_reg: 492312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len); 492412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ppc440spe_dma_fifo_buf); 492512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin return ret; 492612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 492712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 492812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinstatic void __exit ppc440spe_adma_exit(void) 492912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin{ 493012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 493112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_poly); 493212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 493312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_enable); 493412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin driver_remove_file(&ppc440spe_adma_driver.driver, 493512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin &driver_attr_devices); 4936000061245a6797d542854106463b6b20fbdcb12eGrant Likely platform_driver_unregister(&ppc440spe_adma_driver); 493712458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len); 493812458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin kfree(ppc440spe_dma_fifo_buf); 493912458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin} 494012458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 494112458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinarch_initcall(ppc440spe_adma_init); 494212458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschinmodule_exit(ppc440spe_adma_exit); 494312458ea06efd7b44281e68fe59c950ec7d59c649Anatolij Gustschin 494412458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinMODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>"); 494512458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinMODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver"); 494612458ea06efd7b44281e68fe59c950ec7d59c649Anatolij GustschinMODULE_LICENSE("GPL"); 4947