19c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips/* 29c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * talitos - Freescale Integrated Security Engine (SEC) device driver 39c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * 45228f0f79e983c2b39c202c75af901ceb0003fc1Kim Phillips * Copyright (c) 2008-2011 Freescale Semiconductor, Inc. 59c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * 69c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * Scatterlist Crypto API glue code copied from files with the following: 79c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au> 89c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * 99c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * Crypto algorithm registration code copied from hifn driver: 109c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru> 119c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * All rights reserved. 129c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * 139c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * This program is free software; you can redistribute it and/or modify 149c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * it under the terms of the GNU General Public License as published by 159c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * the Free Software Foundation; either version 2 of the License, or 169c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * (at your option) any later version. 179c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * 189c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * This program is distributed in the hope that it will be useful, 199c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * but WITHOUT ANY WARRANTY; without even the implied warranty of 209c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 219c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * GNU General Public License for more details. 229c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * 239c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * You should have received a copy of the GNU General Public License 249c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * along with this program; if not, write to the Free Software 259c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 269c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 279c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 289c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#include <linux/kernel.h> 299c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#include <linux/module.h> 309c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#include <linux/mod_devicetable.h> 319c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#include <linux/device.h> 329c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#include <linux/interrupt.h> 339c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#include <linux/crypto.h> 349c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#include <linux/hw_random.h> 359c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#include <linux/of_platform.h> 369c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#include <linux/dma-mapping.h> 379c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#include <linux/io.h> 389c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#include <linux/spinlock.h> 399c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#include <linux/rtnetlink.h> 405a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 419c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 429c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#include <crypto/algapi.h> 439c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#include <crypto/aes.h> 443952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper#include <crypto/des.h> 459c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#include <crypto/sha.h> 46497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper#include <crypto/md5.h> 479c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#include <crypto/aead.h> 489c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#include <crypto/authenc.h> 494de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper#include <crypto/skcipher.h> 50acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper#include <crypto/hash.h> 51acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper#include <crypto/internal/hash.h> 524de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper#include <crypto/scatterwalk.h> 539c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 549c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#include "talitos.h" 559c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 569c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#define TALITOS_TIMEOUT 100000 579c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#define TALITOS_MAX_DATA_LEN 65535 589c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 599c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#define DESC_TYPE(desc_hdr) ((be32_to_cpu(desc_hdr) >> 3) & 0x1f) 609c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#define PRIMARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 28) & 0xf) 619c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#define SECONDARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 16) & 0xf) 629c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 639c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips/* descriptor pointer entry */ 649c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstruct talitos_ptr { 659c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips __be16 len; /* length */ 669c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips u8 j_extent; /* jump to sg link table and/or extent */ 679c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips u8 eptr; /* extended address */ 689c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips __be32 ptr; /* address */ 699c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips}; 709c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 71497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipperstatic const struct talitos_ptr zero_entry = { 72497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .len = 0, 73497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .j_extent = 0, 74497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .eptr = 0, 75497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .ptr = 0 76497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper}; 77497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 789c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips/* descriptor */ 799c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstruct talitos_desc { 809c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips __be32 hdr; /* header high bits */ 819c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips __be32 hdr_lo; /* header low bits */ 829c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_ptr ptr[7]; /* ptr/len pair array */ 839c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips}; 849c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 859c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips/** 869c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * talitos_request - descriptor submission request 879c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * @desc: descriptor pointer (kernel virtual) 889c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * @dma_desc: descriptor's physical bus address 899c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * @callback: whom to call when descriptor processing is done 909c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * @context: caller context (optional) 919c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 929c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstruct talitos_request { 939c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_desc *desc; 949c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dma_addr_t dma_desc; 959c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips void (*callback) (struct device *dev, struct talitos_desc *desc, 969c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips void *context, int error); 979c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips void *context; 989c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips}; 999c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 1004b992628812137e30cda3510510cf3c052345b30Kim Phillips/* per-channel fifo management */ 1014b992628812137e30cda3510510cf3c052345b30Kim Phillipsstruct talitos_channel { 102ad42d5fc85383278663ecb58a24f6547ad0ba735Kim Phillips void __iomem *reg; 103ad42d5fc85383278663ecb58a24f6547ad0ba735Kim Phillips 1044b992628812137e30cda3510510cf3c052345b30Kim Phillips /* request fifo */ 1054b992628812137e30cda3510510cf3c052345b30Kim Phillips struct talitos_request *fifo; 1064b992628812137e30cda3510510cf3c052345b30Kim Phillips 1074b992628812137e30cda3510510cf3c052345b30Kim Phillips /* number of requests pending in channel h/w fifo */ 1084b992628812137e30cda3510510cf3c052345b30Kim Phillips atomic_t submit_count ____cacheline_aligned; 1094b992628812137e30cda3510510cf3c052345b30Kim Phillips 1104b992628812137e30cda3510510cf3c052345b30Kim Phillips /* request submission (head) lock */ 1114b992628812137e30cda3510510cf3c052345b30Kim Phillips spinlock_t head_lock ____cacheline_aligned; 1124b992628812137e30cda3510510cf3c052345b30Kim Phillips /* index to next free descriptor request */ 1134b992628812137e30cda3510510cf3c052345b30Kim Phillips int head; 1144b992628812137e30cda3510510cf3c052345b30Kim Phillips 1154b992628812137e30cda3510510cf3c052345b30Kim Phillips /* request release (tail) lock */ 1164b992628812137e30cda3510510cf3c052345b30Kim Phillips spinlock_t tail_lock ____cacheline_aligned; 1174b992628812137e30cda3510510cf3c052345b30Kim Phillips /* index to next in-progress/done descriptor request */ 1184b992628812137e30cda3510510cf3c052345b30Kim Phillips int tail; 1194b992628812137e30cda3510510cf3c052345b30Kim Phillips}; 1204b992628812137e30cda3510510cf3c052345b30Kim Phillips 1219c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstruct talitos_private { 1229c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct device *dev; 1232dc11581376829303b98eadb2de253bee065a56aGrant Likely struct platform_device *ofdev; 1249c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips void __iomem *reg; 125c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips int irq[2]; 1269c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 127511d63cb19329235bc9298b64010ec494b5e1408Horia Geanta /* SEC global registers lock */ 128511d63cb19329235bc9298b64010ec494b5e1408Horia Geanta spinlock_t reg_lock ____cacheline_aligned; 129511d63cb19329235bc9298b64010ec494b5e1408Horia Geanta 1309c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* SEC version geometry (from device tree node) */ 1319c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unsigned int num_channels; 1329c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unsigned int chfifo_len; 1339c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unsigned int exec_units; 1349c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unsigned int desc_types; 1359c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 136f3c85bc1bc72b4cc8d58664a490a9d42bdb6565aLee Nipper /* SEC Compatibility info */ 137f3c85bc1bc72b4cc8d58664a490a9d42bdb6565aLee Nipper unsigned long features; 138f3c85bc1bc72b4cc8d58664a490a9d42bdb6565aLee Nipper 1399c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* 1409c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * length of the request fifo 1419c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * fifo_len is chfifo_len rounded up to next power of 2 1429c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * so we can use bitwise ops to wrap 1439c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 1449c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unsigned int fifo_len; 1459c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 1464b992628812137e30cda3510510cf3c052345b30Kim Phillips struct talitos_channel *chan; 1479c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 1484b992628812137e30cda3510510cf3c052345b30Kim Phillips /* next channel to be assigned next incoming descriptor */ 1494b992628812137e30cda3510510cf3c052345b30Kim Phillips atomic_t last_chan ____cacheline_aligned; 1509c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 1519c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* request callback tasklet */ 152c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips struct tasklet_struct done_task[2]; 1539c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 1549c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* list of registered algorithms */ 1559c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct list_head alg_list; 1569c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 1579c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* hwrng device */ 1589c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct hwrng rng; 1599c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips}; 1609c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 161f3c85bc1bc72b4cc8d58664a490a9d42bdb6565aLee Nipper/* .features flag */ 162f3c85bc1bc72b4cc8d58664a490a9d42bdb6565aLee Nipper#define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001 163fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips#define TALITOS_FTR_HW_AUTH_CHECK 0x00000002 16460f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips#define TALITOS_FTR_SHA224_HWINIT 0x00000004 16579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper#define TALITOS_FTR_HMAC_OK 0x00000008 166f3c85bc1bc72b4cc8d58664a490a9d42bdb6565aLee Nipper 16781eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillipsstatic void to_talitos_ptr(struct talitos_ptr *talitos_ptr, dma_addr_t dma_addr) 16881eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips{ 16981eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips talitos_ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr)); 170a752447af5b61f19e9c50322d9b07cea9a086084Kim Phillips talitos_ptr->eptr = upper_32_bits(dma_addr); 17181eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips} 17281eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips 1739c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips/* 1749c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * map virtual single (contiguous) pointer to h/w descriptor pointer 1759c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 1769c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstatic void map_single_talitos_ptr(struct device *dev, 1779c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_ptr *talitos_ptr, 1789c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unsigned short len, void *data, 1799c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unsigned char extent, 1809c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips enum dma_data_direction dir) 1819c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 18281eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips dma_addr_t dma_addr = dma_map_single(dev, data, len, dir); 18381eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips 1849c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips talitos_ptr->len = cpu_to_be16(len); 18581eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips to_talitos_ptr(talitos_ptr, dma_addr); 1869c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips talitos_ptr->j_extent = extent; 1879c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 1889c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 1899c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips/* 1909c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * unmap bus single (contiguous) h/w descriptor pointer 1919c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 1929c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstatic void unmap_single_talitos_ptr(struct device *dev, 1939c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_ptr *talitos_ptr, 1949c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips enum dma_data_direction dir) 1959c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 1969c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dma_unmap_single(dev, be32_to_cpu(talitos_ptr->ptr), 1979c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips be16_to_cpu(talitos_ptr->len), dir); 1989c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 1999c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2009c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstatic int reset_channel(struct device *dev, int ch) 2019c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 2029c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_private *priv = dev_get_drvdata(dev); 2039c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unsigned int timeout = TALITOS_TIMEOUT; 2049c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 205ad42d5fc85383278663ecb58a24f6547ad0ba735Kim Phillips setbits32(priv->chan[ch].reg + TALITOS_CCCR, TALITOS_CCCR_RESET); 2069c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 207ad42d5fc85383278663ecb58a24f6547ad0ba735Kim Phillips while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & TALITOS_CCCR_RESET) 2089c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips && --timeout) 2099c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips cpu_relax(); 2109c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2119c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (timeout == 0) { 2129c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "failed to reset channel %d\n", ch); 2139c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return -EIO; 2149c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 2159c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 21681eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips /* set 36-bit addressing, done writeback enable and done IRQ enable */ 217ad42d5fc85383278663ecb58a24f6547ad0ba735Kim Phillips setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE | 21881eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE); 2199c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 220fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips /* and ICCR writeback, if available */ 221fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) 222ad42d5fc85383278663ecb58a24f6547ad0ba735Kim Phillips setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 223fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips TALITOS_CCCR_LO_IWSE); 224fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips 2259c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return 0; 2269c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 2279c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2289c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstatic int reset_device(struct device *dev) 2299c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 2309c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_private *priv = dev_get_drvdata(dev); 2319c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unsigned int timeout = TALITOS_TIMEOUT; 232c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips u32 mcr = TALITOS_MCR_SWR; 2339c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 234c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips setbits32(priv->reg + TALITOS_MCR, mcr); 2359c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2369c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips while ((in_be32(priv->reg + TALITOS_MCR) & TALITOS_MCR_SWR) 2379c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips && --timeout) 2389c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips cpu_relax(); 2399c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2402cdba3cf6ffc1fbf880a6fbfa9e7bb757e3d6526Kim Phillips if (priv->irq[1]) { 241c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3; 242c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips setbits32(priv->reg + TALITOS_MCR, mcr); 243c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips } 244c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips 2459c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (timeout == 0) { 2469c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "failed to reset device\n"); 2479c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return -EIO; 2489c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 2499c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2509c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return 0; 2519c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 2529c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2539c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips/* 2549c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * Reset and initialize the device 2559c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 2569c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstatic int init_device(struct device *dev) 2579c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 2589c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_private *priv = dev_get_drvdata(dev); 2599c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips int ch, err; 2609c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2619c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* 2629c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * Master reset 2639c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * errata documentation: warning: certain SEC interrupts 2649c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * are not fully cleared by writing the MCR:SWR bit, 2659c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * set bit twice to completely reset 2669c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 2679c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips err = reset_device(dev); 2689c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (err) 2699c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return err; 2709c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2719c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips err = reset_device(dev); 2729c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (err) 2739c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return err; 2749c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2759c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* reset channels */ 2769c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips for (ch = 0; ch < priv->num_channels; ch++) { 2779c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips err = reset_channel(dev, ch); 2789c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (err) 2799c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return err; 2809c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 2819c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2829c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* enable channel done and error interrupts */ 2839c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips setbits32(priv->reg + TALITOS_IMR, TALITOS_IMR_INIT); 2849c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT); 2859c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 286fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips /* disable integrity check error interrupts (use writeback instead) */ 287fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) 288fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips setbits32(priv->reg + TALITOS_MDEUICR_LO, 289fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips TALITOS_MDEUICR_LO_ICE); 290fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips 2919c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return 0; 2929c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 2939c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2949c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips/** 2959c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * talitos_submit - submits a descriptor to the device for processing 2969c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * @dev: the SEC device to be used 2975228f0f79e983c2b39c202c75af901ceb0003fc1Kim Phillips * @ch: the SEC device channel to be used 2989c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * @desc: the descriptor to be processed by the device 2999c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * @callback: whom to call when processing is complete 3009c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * @context: a handle for use by caller (optional) 3019c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * 3029c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * desc must contain valid dma-mapped (bus physical) address pointers. 3039c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * callback must check err and feedback in descriptor header 3049c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * for device processing status. 3059c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 3065228f0f79e983c2b39c202c75af901ceb0003fc1Kim Phillipsstatic int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, 3079c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips void (*callback)(struct device *dev, 3089c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_desc *desc, 3099c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips void *context, int error), 3109c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips void *context) 3119c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 3129c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_private *priv = dev_get_drvdata(dev); 3139c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_request *request; 3145228f0f79e983c2b39c202c75af901ceb0003fc1Kim Phillips unsigned long flags; 3159c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips int head; 3169c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 3174b992628812137e30cda3510510cf3c052345b30Kim Phillips spin_lock_irqsave(&priv->chan[ch].head_lock, flags); 3189c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 3194b992628812137e30cda3510510cf3c052345b30Kim Phillips if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) { 320ec6644d6325b5a38525f1d5b20fd4bf7db05cf2aKim Phillips /* h/w fifo is full */ 3214b992628812137e30cda3510510cf3c052345b30Kim Phillips spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags); 3229c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return -EAGAIN; 3239c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 3249c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 3254b992628812137e30cda3510510cf3c052345b30Kim Phillips head = priv->chan[ch].head; 3264b992628812137e30cda3510510cf3c052345b30Kim Phillips request = &priv->chan[ch].fifo[head]; 327ec6644d6325b5a38525f1d5b20fd4bf7db05cf2aKim Phillips 3289c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* map descriptor and save caller data */ 3299c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips request->dma_desc = dma_map_single(dev, desc, sizeof(*desc), 3309c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips DMA_BIDIRECTIONAL); 3319c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips request->callback = callback; 3329c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips request->context = context; 3339c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 3349c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* increment fifo head */ 3354b992628812137e30cda3510510cf3c052345b30Kim Phillips priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1); 3369c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 3379c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips smp_wmb(); 3389c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips request->desc = desc; 3399c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 3409c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* GO! */ 3419c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips wmb(); 342ad42d5fc85383278663ecb58a24f6547ad0ba735Kim Phillips out_be32(priv->chan[ch].reg + TALITOS_FF, 343ad42d5fc85383278663ecb58a24f6547ad0ba735Kim Phillips upper_32_bits(request->dma_desc)); 344ad42d5fc85383278663ecb58a24f6547ad0ba735Kim Phillips out_be32(priv->chan[ch].reg + TALITOS_FF_LO, 345a752447af5b61f19e9c50322d9b07cea9a086084Kim Phillips lower_32_bits(request->dma_desc)); 3469c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 3474b992628812137e30cda3510510cf3c052345b30Kim Phillips spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags); 3489c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 3499c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return -EINPROGRESS; 3509c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 3519c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 3529c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips/* 3539c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * process what was done, notify callback of error if not 3549c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 3559c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstatic void flush_channel(struct device *dev, int ch, int error, int reset_ch) 3569c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 3579c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_private *priv = dev_get_drvdata(dev); 3589c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_request *request, saved_req; 3599c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unsigned long flags; 3609c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips int tail, status; 3619c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 3624b992628812137e30cda3510510cf3c052345b30Kim Phillips spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); 3639c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 3644b992628812137e30cda3510510cf3c052345b30Kim Phillips tail = priv->chan[ch].tail; 3654b992628812137e30cda3510510cf3c052345b30Kim Phillips while (priv->chan[ch].fifo[tail].desc) { 3664b992628812137e30cda3510510cf3c052345b30Kim Phillips request = &priv->chan[ch].fifo[tail]; 3679c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 3689c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* descriptors with their done bits set don't get the error */ 3699c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips rmb(); 370ca38a814c6f86db0aa58884a31093d4f096e20aaLee Nipper if ((request->desc->hdr & DESC_HDR_DONE) == DESC_HDR_DONE) 3719c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips status = 0; 372ca38a814c6f86db0aa58884a31093d4f096e20aaLee Nipper else 3739c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (!error) 3749c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips break; 3759c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips else 3769c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips status = error; 3779c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 3789c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dma_unmap_single(dev, request->dma_desc, 379e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips sizeof(struct talitos_desc), 380e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips DMA_BIDIRECTIONAL); 3819c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 3829c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* copy entries so we can call callback outside lock */ 3839c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips saved_req.desc = request->desc; 3849c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips saved_req.callback = request->callback; 3859c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips saved_req.context = request->context; 3869c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 3879c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* release request entry in fifo */ 3889c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips smp_wmb(); 3899c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips request->desc = NULL; 3909c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 3919c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* increment fifo tail */ 3924b992628812137e30cda3510510cf3c052345b30Kim Phillips priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1); 3939c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 3944b992628812137e30cda3510510cf3c052345b30Kim Phillips spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags); 395ec6644d6325b5a38525f1d5b20fd4bf7db05cf2aKim Phillips 3964b992628812137e30cda3510510cf3c052345b30Kim Phillips atomic_dec(&priv->chan[ch].submit_count); 397ec6644d6325b5a38525f1d5b20fd4bf7db05cf2aKim Phillips 3989c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips saved_req.callback(dev, saved_req.desc, saved_req.context, 3999c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips status); 4009c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* channel may resume processing in single desc error case */ 4019c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (error && !reset_ch && status == error) 4029c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return; 4034b992628812137e30cda3510510cf3c052345b30Kim Phillips spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); 4044b992628812137e30cda3510510cf3c052345b30Kim Phillips tail = priv->chan[ch].tail; 4059c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 4069c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 4074b992628812137e30cda3510510cf3c052345b30Kim Phillips spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags); 4089c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 4099c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 4109c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips/* 4119c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * process completed requests for channels that have done status 4129c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 413c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips#define DEF_TALITOS_DONE(name, ch_done_mask) \ 414c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillipsstatic void talitos_done_##name(unsigned long data) \ 415c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips{ \ 416c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips struct device *dev = (struct device *)data; \ 417c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips struct talitos_private *priv = dev_get_drvdata(dev); \ 418511d63cb19329235bc9298b64010ec494b5e1408Horia Geanta unsigned long flags; \ 419c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips \ 420c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips if (ch_done_mask & 1) \ 421c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips flush_channel(dev, 0, 0, 0); \ 422c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips if (priv->num_channels == 1) \ 423c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips goto out; \ 424c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips if (ch_done_mask & (1 << 2)) \ 425c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips flush_channel(dev, 1, 0, 0); \ 426c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips if (ch_done_mask & (1 << 4)) \ 427c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips flush_channel(dev, 2, 0, 0); \ 428c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips if (ch_done_mask & (1 << 6)) \ 429c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips flush_channel(dev, 3, 0, 0); \ 430c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips \ 431c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillipsout: \ 432c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips /* At this point, all completed channels have been processed */ \ 433c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips /* Unmask done interrupts for channels completed later on. */ \ 434511d63cb19329235bc9298b64010ec494b5e1408Horia Geanta spin_lock_irqsave(&priv->reg_lock, flags); \ 435c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 436c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT); \ 437511d63cb19329235bc9298b64010ec494b5e1408Horia Geanta spin_unlock_irqrestore(&priv->reg_lock, flags); \ 4389c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 439c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim PhillipsDEF_TALITOS_DONE(4ch, TALITOS_ISR_4CHDONE) 440c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim PhillipsDEF_TALITOS_DONE(ch0_2, TALITOS_ISR_CH_0_2_DONE) 441c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim PhillipsDEF_TALITOS_DONE(ch1_3, TALITOS_ISR_CH_1_3_DONE) 4429c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 4439c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips/* 4449c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * locate current (offending) descriptor 4459c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 4463e721aeb3df3816e283ab18e327cd4652972e213Kim Phillipsstatic u32 current_desc_hdr(struct device *dev, int ch) 4479c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 4489c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_private *priv = dev_get_drvdata(dev); 4494b992628812137e30cda3510510cf3c052345b30Kim Phillips int tail = priv->chan[ch].tail; 4509c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dma_addr_t cur_desc; 4519c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 452ad42d5fc85383278663ecb58a24f6547ad0ba735Kim Phillips cur_desc = in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO); 4539c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 4544b992628812137e30cda3510510cf3c052345b30Kim Phillips while (priv->chan[ch].fifo[tail].dma_desc != cur_desc) { 4559c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips tail = (tail + 1) & (priv->fifo_len - 1); 4564b992628812137e30cda3510510cf3c052345b30Kim Phillips if (tail == priv->chan[ch].tail) { 4579c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "couldn't locate current descriptor\n"); 4583e721aeb3df3816e283ab18e327cd4652972e213Kim Phillips return 0; 4599c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 4609c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 4619c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 4623e721aeb3df3816e283ab18e327cd4652972e213Kim Phillips return priv->chan[ch].fifo[tail].desc->hdr; 4639c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 4649c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 4659c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips/* 4669c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * user diagnostics; report root cause of error based on execution unit status 4679c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 4683e721aeb3df3816e283ab18e327cd4652972e213Kim Phillipsstatic void report_eu_error(struct device *dev, int ch, u32 desc_hdr) 4699c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 4709c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_private *priv = dev_get_drvdata(dev); 4719c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips int i; 4729c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 4733e721aeb3df3816e283ab18e327cd4652972e213Kim Phillips if (!desc_hdr) 474ad42d5fc85383278663ecb58a24f6547ad0ba735Kim Phillips desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF); 4753e721aeb3df3816e283ab18e327cd4652972e213Kim Phillips 4763e721aeb3df3816e283ab18e327cd4652972e213Kim Phillips switch (desc_hdr & DESC_HDR_SEL0_MASK) { 4779c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips case DESC_HDR_SEL0_AFEU: 4789c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "AFEUISR 0x%08x_%08x\n", 4799c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips in_be32(priv->reg + TALITOS_AFEUISR), 4809c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips in_be32(priv->reg + TALITOS_AFEUISR_LO)); 4819c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips break; 4829c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips case DESC_HDR_SEL0_DEU: 4839c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "DEUISR 0x%08x_%08x\n", 4849c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips in_be32(priv->reg + TALITOS_DEUISR), 4859c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips in_be32(priv->reg + TALITOS_DEUISR_LO)); 4869c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips break; 4879c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips case DESC_HDR_SEL0_MDEUA: 4889c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips case DESC_HDR_SEL0_MDEUB: 4899c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "MDEUISR 0x%08x_%08x\n", 4909c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips in_be32(priv->reg + TALITOS_MDEUISR), 4919c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips in_be32(priv->reg + TALITOS_MDEUISR_LO)); 4929c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips break; 4939c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips case DESC_HDR_SEL0_RNG: 4949c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "RNGUISR 0x%08x_%08x\n", 4959c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips in_be32(priv->reg + TALITOS_RNGUISR), 4969c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips in_be32(priv->reg + TALITOS_RNGUISR_LO)); 4979c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips break; 4989c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips case DESC_HDR_SEL0_PKEU: 4999c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "PKEUISR 0x%08x_%08x\n", 5009c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips in_be32(priv->reg + TALITOS_PKEUISR), 5019c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips in_be32(priv->reg + TALITOS_PKEUISR_LO)); 5029c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips break; 5039c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips case DESC_HDR_SEL0_AESU: 5049c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "AESUISR 0x%08x_%08x\n", 5059c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips in_be32(priv->reg + TALITOS_AESUISR), 5069c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips in_be32(priv->reg + TALITOS_AESUISR_LO)); 5079c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips break; 5089c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips case DESC_HDR_SEL0_CRCU: 5099c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "CRCUISR 0x%08x_%08x\n", 5109c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips in_be32(priv->reg + TALITOS_CRCUISR), 5119c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips in_be32(priv->reg + TALITOS_CRCUISR_LO)); 5129c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips break; 5139c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips case DESC_HDR_SEL0_KEU: 5149c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "KEUISR 0x%08x_%08x\n", 5159c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips in_be32(priv->reg + TALITOS_KEUISR), 5169c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips in_be32(priv->reg + TALITOS_KEUISR_LO)); 5179c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips break; 5189c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 5199c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 5203e721aeb3df3816e283ab18e327cd4652972e213Kim Phillips switch (desc_hdr & DESC_HDR_SEL1_MASK) { 5219c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips case DESC_HDR_SEL1_MDEUA: 5229c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips case DESC_HDR_SEL1_MDEUB: 5239c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "MDEUISR 0x%08x_%08x\n", 5249c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips in_be32(priv->reg + TALITOS_MDEUISR), 5259c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips in_be32(priv->reg + TALITOS_MDEUISR_LO)); 5269c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips break; 5279c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips case DESC_HDR_SEL1_CRCU: 5289c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "CRCUISR 0x%08x_%08x\n", 5299c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips in_be32(priv->reg + TALITOS_CRCUISR), 5309c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips in_be32(priv->reg + TALITOS_CRCUISR_LO)); 5319c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips break; 5329c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 5339c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 5349c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips for (i = 0; i < 8; i++) 5359c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "DESCBUF 0x%08x_%08x\n", 536ad42d5fc85383278663ecb58a24f6547ad0ba735Kim Phillips in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i), 537ad42d5fc85383278663ecb58a24f6547ad0ba735Kim Phillips in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i)); 5389c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 5399c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 5409c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips/* 5419c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * recover from error interrupts 5429c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 5435e718a09e5cafc8922f3fe22206423449a2801c9Kim Phillipsstatic void talitos_error(struct device *dev, u32 isr, u32 isr_lo) 5449c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 5459c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_private *priv = dev_get_drvdata(dev); 5469c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unsigned int timeout = TALITOS_TIMEOUT; 5479c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips int ch, error, reset_dev = 0, reset_ch = 0; 54840405f10b853ef404b395c6febca71b0de185490Kim Phillips u32 v, v_lo; 5499c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 5509c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips for (ch = 0; ch < priv->num_channels; ch++) { 5519c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* skip channels without errors */ 5529c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (!(isr & (1 << (ch * 2 + 1)))) 5539c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips continue; 5549c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 5559c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips error = -EINVAL; 5569c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 557ad42d5fc85383278663ecb58a24f6547ad0ba735Kim Phillips v = in_be32(priv->chan[ch].reg + TALITOS_CCPSR); 558ad42d5fc85383278663ecb58a24f6547ad0ba735Kim Phillips v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO); 5599c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 5609c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (v_lo & TALITOS_CCPSR_LO_DOF) { 5619c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "double fetch fifo overflow error\n"); 5629c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips error = -EAGAIN; 5639c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips reset_ch = 1; 5649c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 5659c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (v_lo & TALITOS_CCPSR_LO_SOF) { 5669c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* h/w dropped descriptor */ 5679c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "single fetch fifo overflow error\n"); 5689c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips error = -EAGAIN; 5699c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 5709c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (v_lo & TALITOS_CCPSR_LO_MDTE) 5719c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "master data transfer error\n"); 5729c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (v_lo & TALITOS_CCPSR_LO_SGDLZ) 5739c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "s/g data length zero error\n"); 5749c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (v_lo & TALITOS_CCPSR_LO_FPZ) 5759c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "fetch pointer zero error\n"); 5769c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (v_lo & TALITOS_CCPSR_LO_IDH) 5779c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "illegal descriptor header error\n"); 5789c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (v_lo & TALITOS_CCPSR_LO_IEU) 5799c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "invalid execution unit error\n"); 5809c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (v_lo & TALITOS_CCPSR_LO_EU) 5813e721aeb3df3816e283ab18e327cd4652972e213Kim Phillips report_eu_error(dev, ch, current_desc_hdr(dev, ch)); 5829c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (v_lo & TALITOS_CCPSR_LO_GB) 5839c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "gather boundary error\n"); 5849c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (v_lo & TALITOS_CCPSR_LO_GRL) 5859c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "gather return/length error\n"); 5869c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (v_lo & TALITOS_CCPSR_LO_SB) 5879c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "scatter boundary error\n"); 5889c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (v_lo & TALITOS_CCPSR_LO_SRL) 5899c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "scatter return/length error\n"); 5909c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 5919c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips flush_channel(dev, ch, error, reset_ch); 5929c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 5939c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (reset_ch) { 5949c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips reset_channel(dev, ch); 5959c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } else { 596ad42d5fc85383278663ecb58a24f6547ad0ba735Kim Phillips setbits32(priv->chan[ch].reg + TALITOS_CCCR, 5979c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips TALITOS_CCCR_CONT); 598ad42d5fc85383278663ecb58a24f6547ad0ba735Kim Phillips setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0); 599ad42d5fc85383278663ecb58a24f6547ad0ba735Kim Phillips while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & 6009c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips TALITOS_CCCR_CONT) && --timeout) 6019c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips cpu_relax(); 6029c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (timeout == 0) { 6039c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "failed to restart channel %d\n", 6049c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips ch); 6059c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips reset_dev = 1; 6069c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 6079c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 6089c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 609c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips if (reset_dev || isr & ~TALITOS_ISR_4CHERR || isr_lo) { 6109c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "done overflow, internal time out, or rngu error: " 6119c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips "ISR 0x%08x_%08x\n", isr, isr_lo); 6129c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 6139c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* purge request queues */ 6149c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips for (ch = 0; ch < priv->num_channels; ch++) 6159c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips flush_channel(dev, ch, -EIO, 1); 6169c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 6179c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* reset and reinitialize the device */ 6189c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips init_device(dev); 6199c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 6209c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 6219c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 622c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips#define DEF_TALITOS_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \ 623c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillipsstatic irqreturn_t talitos_interrupt_##name(int irq, void *data) \ 624c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips{ \ 625c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips struct device *dev = data; \ 626c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips struct talitos_private *priv = dev_get_drvdata(dev); \ 627c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips u32 isr, isr_lo; \ 628511d63cb19329235bc9298b64010ec494b5e1408Horia Geanta unsigned long flags; \ 629c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips \ 630511d63cb19329235bc9298b64010ec494b5e1408Horia Geanta spin_lock_irqsave(&priv->reg_lock, flags); \ 631c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips isr = in_be32(priv->reg + TALITOS_ISR); \ 632c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \ 633c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips /* Acknowledge interrupt */ \ 634c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \ 635c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \ 636c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips \ 637511d63cb19329235bc9298b64010ec494b5e1408Horia Geanta if (unlikely(isr & ch_err_mask || isr_lo)) { \ 638511d63cb19329235bc9298b64010ec494b5e1408Horia Geanta spin_unlock_irqrestore(&priv->reg_lock, flags); \ 639511d63cb19329235bc9298b64010ec494b5e1408Horia Geanta talitos_error(dev, isr & ch_err_mask, isr_lo); \ 640511d63cb19329235bc9298b64010ec494b5e1408Horia Geanta } \ 641511d63cb19329235bc9298b64010ec494b5e1408Horia Geanta else { \ 642c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips if (likely(isr & ch_done_mask)) { \ 643c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips /* mask further done interrupts. */ \ 644c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 645c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips /* done_task will unmask done interrupts at exit */ \ 646c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips tasklet_schedule(&priv->done_task[tlet]); \ 647c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips } \ 648511d63cb19329235bc9298b64010ec494b5e1408Horia Geanta spin_unlock_irqrestore(&priv->reg_lock, flags); \ 649511d63cb19329235bc9298b64010ec494b5e1408Horia Geanta } \ 650c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips \ 651c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \ 652c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips IRQ_NONE; \ 6539c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 654c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim PhillipsDEF_TALITOS_INTERRUPT(4ch, TALITOS_ISR_4CHDONE, TALITOS_ISR_4CHERR, 0) 655c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim PhillipsDEF_TALITOS_INTERRUPT(ch0_2, TALITOS_ISR_CH_0_2_DONE, TALITOS_ISR_CH_0_2_ERR, 0) 656c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim PhillipsDEF_TALITOS_INTERRUPT(ch1_3, TALITOS_ISR_CH_1_3_DONE, TALITOS_ISR_CH_1_3_ERR, 1) 6579c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 6589c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips/* 6599c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * hwrng 6609c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 6619c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstatic int talitos_rng_data_present(struct hwrng *rng, int wait) 6629c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 6639c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct device *dev = (struct device *)rng->priv; 6649c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_private *priv = dev_get_drvdata(dev); 6659c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips u32 ofl; 6669c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips int i; 6679c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 6689c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips for (i = 0; i < 20; i++) { 6699c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips ofl = in_be32(priv->reg + TALITOS_RNGUSR_LO) & 6709c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips TALITOS_RNGUSR_LO_OFL; 6719c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (ofl || !wait) 6729c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips break; 6739c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips udelay(10); 6749c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 6759c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 6769c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return !!ofl; 6779c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 6789c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 6799c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstatic int talitos_rng_data_read(struct hwrng *rng, u32 *data) 6809c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 6819c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct device *dev = (struct device *)rng->priv; 6829c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_private *priv = dev_get_drvdata(dev); 6839c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 6849c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* rng fifo requires 64-bit accesses */ 6859c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips *data = in_be32(priv->reg + TALITOS_RNGU_FIFO); 6869c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips *data = in_be32(priv->reg + TALITOS_RNGU_FIFO_LO); 6879c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 6889c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return sizeof(u32); 6899c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 6909c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 6919c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstatic int talitos_rng_init(struct hwrng *rng) 6929c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 6939c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct device *dev = (struct device *)rng->priv; 6949c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_private *priv = dev_get_drvdata(dev); 6959c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unsigned int timeout = TALITOS_TIMEOUT; 6969c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 6979c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips setbits32(priv->reg + TALITOS_RNGURCR_LO, TALITOS_RNGURCR_LO_SR); 6989c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips while (!(in_be32(priv->reg + TALITOS_RNGUSR_LO) & TALITOS_RNGUSR_LO_RD) 6999c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips && --timeout) 7009c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips cpu_relax(); 7019c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (timeout == 0) { 7029c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "failed to reset rng hw\n"); 7039c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return -ENODEV; 7049c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 7059c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 7069c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* start generating */ 7079c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips setbits32(priv->reg + TALITOS_RNGUDSR_LO, 0); 7089c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 7099c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return 0; 7109c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 7119c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 7129c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstatic int talitos_register_rng(struct device *dev) 7139c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 7149c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_private *priv = dev_get_drvdata(dev); 7159c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 7169c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips priv->rng.name = dev_driver_string(dev), 7179c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips priv->rng.init = talitos_rng_init, 7189c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips priv->rng.data_present = talitos_rng_data_present, 7199c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips priv->rng.data_read = talitos_rng_data_read, 7209c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips priv->rng.priv = (unsigned long)dev; 7219c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 7229c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return hwrng_register(&priv->rng); 7239c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 7249c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 7259c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstatic void talitos_unregister_rng(struct device *dev) 7269c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 7279c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_private *priv = dev_get_drvdata(dev); 7289c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 7299c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips hwrng_unregister(&priv->rng); 7309c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 7319c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 7329c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips/* 7339c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * crypto alg 7349c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 7359c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#define TALITOS_CRA_PRIORITY 3000 7369c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips#define TALITOS_MAX_KEY_SIZE 64 7373952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ 73870bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper 739497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper#define MD5_BLOCK_SIZE 64 7409c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 7419c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstruct talitos_ctx { 7429c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct device *dev; 7435228f0f79e983c2b39c202c75af901ceb0003fc1Kim Phillips int ch; 7449c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips __be32 desc_hdr_template; 7459c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips u8 key[TALITOS_MAX_KEY_SIZE]; 74670bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper u8 iv[TALITOS_MAX_IV_LENGTH]; 7479c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unsigned int keylen; 7489c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unsigned int enckeylen; 7499c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unsigned int authkeylen; 7509c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unsigned int authsize; 7519c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips}; 7529c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 753497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper#define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE 754497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper#define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512 755497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 756497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipperstruct talitos_ahash_req_ctx { 75760f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; 758497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper unsigned int hw_context_size; 759497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper u8 buf[HASH_MAX_BLOCK_SIZE]; 760497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper u8 bufnext[HASH_MAX_BLOCK_SIZE]; 76160f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips unsigned int swinit; 762497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper unsigned int first; 763497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper unsigned int last; 764497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper unsigned int to_hash_later; 7655e833bc4166ea524c00a95e777e4344844ed189fLee Nipper u64 nbuf; 766497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct scatterlist bufsl[2]; 767497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct scatterlist *psrc; 768497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper}; 769497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 77056af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipperstatic int aead_setauthsize(struct crypto_aead *authenc, 77156af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper unsigned int authsize) 7729c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 7739c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 7749c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 7759c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips ctx->authsize = authsize; 7769c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 7779c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return 0; 7789c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 7799c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 78056af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipperstatic int aead_setkey(struct crypto_aead *authenc, 78156af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper const u8 *key, unsigned int keylen) 7829c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 7839c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 7849c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct rtattr *rta = (void *)key; 7859c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct crypto_authenc_key_param *param; 7869c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unsigned int authkeylen; 7879c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unsigned int enckeylen; 7889c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 7899c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (!RTA_OK(rta, keylen)) 7909c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips goto badkey; 7919c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 7929c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM) 7939c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips goto badkey; 7949c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 7959c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (RTA_PAYLOAD(rta) < sizeof(*param)) 7969c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips goto badkey; 7979c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 7989c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips param = RTA_DATA(rta); 7999c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips enckeylen = be32_to_cpu(param->enckeylen); 8009c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 8019c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips key += RTA_ALIGN(rta->rta_len); 8029c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips keylen -= RTA_ALIGN(rta->rta_len); 8039c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 8049c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (keylen < enckeylen) 8059c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips goto badkey; 8069c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 8079c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips authkeylen = keylen - enckeylen; 8089c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 8099c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (keylen > TALITOS_MAX_KEY_SIZE) 8109c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips goto badkey; 8119c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 8129c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips memcpy(&ctx->key, key, keylen); 8139c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 8149c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips ctx->keylen = keylen; 8159c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips ctx->enckeylen = enckeylen; 8169c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips ctx->authkeylen = authkeylen; 8179c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 8189c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return 0; 8199c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 8209c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsbadkey: 8219c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN); 8229c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return -EINVAL; 8239c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 8249c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 8259c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips/* 82656af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper * talitos_edesc - s/w-extended descriptor 8279c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * @src_nents: number of segments in input scatterlist 8289c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * @dst_nents: number of segments in output scatterlist 8299c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * @dma_len: length of dma mapped link_tbl space 8309c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * @dma_link_tbl: bus physical address of link_tbl 8319c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * @desc: h/w descriptor 8329c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) 8339c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * 8349c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * if decrypting (with authcheck), or either one of src_nents or dst_nents 8359c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * is greater than 1, an integrity check value is concatenated to the end 8369c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * of link_tbl data 8379c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 83856af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipperstruct talitos_edesc { 8399c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips int src_nents; 8409c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips int dst_nents; 8414de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper int src_is_chained; 8424de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper int dst_is_chained; 8439c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips int dma_len; 8449c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dma_addr_t dma_link_tbl; 8459c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_desc desc; 8469c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_ptr link_tbl[0]; 8479c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips}; 8489c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 8494de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipperstatic int talitos_map_sg(struct device *dev, struct scatterlist *sg, 8504de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper unsigned int nents, enum dma_data_direction dir, 8514de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper int chained) 8524de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper{ 8534de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper if (unlikely(chained)) 8544de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper while (sg) { 8554de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper dma_map_sg(dev, sg, 1, dir); 8564de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper sg = scatterwalk_sg_next(sg); 8574de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper } 8584de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper else 8594de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper dma_map_sg(dev, sg, nents, dir); 8604de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper return nents; 8614de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper} 8624de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 8634de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipperstatic void talitos_unmap_sg_chain(struct device *dev, struct scatterlist *sg, 8644de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper enum dma_data_direction dir) 8654de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper{ 8664de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper while (sg) { 8674de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper dma_unmap_sg(dev, sg, 1, dir); 8684de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper sg = scatterwalk_sg_next(sg); 8694de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper } 8704de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper} 8714de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 8724de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipperstatic void talitos_sg_unmap(struct device *dev, 8734de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct talitos_edesc *edesc, 8744de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct scatterlist *src, 8754de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct scatterlist *dst) 8764de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper{ 8774de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper unsigned int src_nents = edesc->src_nents ? : 1; 8784de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper unsigned int dst_nents = edesc->dst_nents ? : 1; 8794de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 8804de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper if (src != dst) { 8814de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper if (edesc->src_is_chained) 8824de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper talitos_unmap_sg_chain(dev, src, DMA_TO_DEVICE); 8834de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper else 8844de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 8854de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 886497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper if (dst) { 887497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper if (edesc->dst_is_chained) 888497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper talitos_unmap_sg_chain(dev, dst, 889497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper DMA_FROM_DEVICE); 890497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper else 891497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper dma_unmap_sg(dev, dst, dst_nents, 892497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper DMA_FROM_DEVICE); 893497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper } 8944de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper } else 8954de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper if (edesc->src_is_chained) 8964de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper talitos_unmap_sg_chain(dev, src, DMA_BIDIRECTIONAL); 8974de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper else 8984de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 8994de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper} 9004de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 9019c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstatic void ipsec_esp_unmap(struct device *dev, 90256af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper struct talitos_edesc *edesc, 9039c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct aead_request *areq) 9049c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 9059c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE); 9069c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE); 9079c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE); 9089c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE); 9099c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 9109c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dma_unmap_sg(dev, areq->assoc, 1, DMA_TO_DEVICE); 9119c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 9124de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper talitos_sg_unmap(dev, edesc, areq->src, areq->dst); 9139c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 9149c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (edesc->dma_len) 9159c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, 9169c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips DMA_BIDIRECTIONAL); 9179c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 9189c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 9199c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips/* 9209c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * ipsec_esp descriptor callbacks 9219c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 9229c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstatic void ipsec_esp_encrypt_done(struct device *dev, 9239c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_desc *desc, void *context, 9249c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips int err) 9259c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 9269c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct aead_request *areq = context; 9279c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct crypto_aead *authenc = crypto_aead_reqtfm(areq); 9289c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 92919bbbc635523703ece28409e59694d5b512b819eKim Phillips struct talitos_edesc *edesc; 9309c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct scatterlist *sg; 9319c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips void *icvdata; 9329c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 93319bbbc635523703ece28409e59694d5b512b819eKim Phillips edesc = container_of(desc, struct talitos_edesc, desc); 93419bbbc635523703ece28409e59694d5b512b819eKim Phillips 9359c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips ipsec_esp_unmap(dev, edesc, areq); 9369c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 9379c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* copy the generated ICV to dst */ 9389c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (edesc->dma_len) { 9399c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips icvdata = &edesc->link_tbl[edesc->src_nents + 940f3c85bc1bc72b4cc8d58664a490a9d42bdb6565aLee Nipper edesc->dst_nents + 2]; 9419c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips sg = sg_last(areq->dst, edesc->dst_nents); 9429c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips memcpy((char *)sg_virt(sg) + sg->length - ctx->authsize, 9439c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips icvdata, ctx->authsize); 9449c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 9459c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 9469c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips kfree(edesc); 9479c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 9489c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips aead_request_complete(areq, err); 9499c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 9509c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 951fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillipsstatic void ipsec_esp_decrypt_swauth_done(struct device *dev, 952e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips struct talitos_desc *desc, 953e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips void *context, int err) 9549c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 9559c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct aead_request *req = context; 9569c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct crypto_aead *authenc = crypto_aead_reqtfm(req); 9579c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 95819bbbc635523703ece28409e59694d5b512b819eKim Phillips struct talitos_edesc *edesc; 9599c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct scatterlist *sg; 9609c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips void *icvdata; 9619c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 96219bbbc635523703ece28409e59694d5b512b819eKim Phillips edesc = container_of(desc, struct talitos_edesc, desc); 96319bbbc635523703ece28409e59694d5b512b819eKim Phillips 9649c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips ipsec_esp_unmap(dev, edesc, req); 9659c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 9669c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (!err) { 9679c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* auth check */ 9689c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (edesc->dma_len) 9699c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips icvdata = &edesc->link_tbl[edesc->src_nents + 970f3c85bc1bc72b4cc8d58664a490a9d42bdb6565aLee Nipper edesc->dst_nents + 2]; 9719c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips else 9729c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips icvdata = &edesc->link_tbl[0]; 9739c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 9749c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips sg = sg_last(req->dst, edesc->dst_nents ? : 1); 9759c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips err = memcmp(icvdata, (char *)sg_virt(sg) + sg->length - 9769c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips ctx->authsize, ctx->authsize) ? -EBADMSG : 0; 9779c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 9789c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 9799c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips kfree(edesc); 9809c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 9819c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips aead_request_complete(req, err); 9829c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 9839c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 984fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillipsstatic void ipsec_esp_decrypt_hwauth_done(struct device *dev, 985e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips struct talitos_desc *desc, 986e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips void *context, int err) 987fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips{ 988fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips struct aead_request *req = context; 98919bbbc635523703ece28409e59694d5b512b819eKim Phillips struct talitos_edesc *edesc; 99019bbbc635523703ece28409e59694d5b512b819eKim Phillips 99119bbbc635523703ece28409e59694d5b512b819eKim Phillips edesc = container_of(desc, struct talitos_edesc, desc); 992fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips 993fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips ipsec_esp_unmap(dev, edesc, req); 994fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips 995fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips /* check ICV auth status */ 996e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) != 997e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips DESC_HDR_LO_ICCR1_PASS)) 998e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips err = -EBADMSG; 999fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips 1000fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips kfree(edesc); 1001fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips 1002fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips aead_request_complete(req, err); 1003fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips} 1004fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips 10059c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips/* 10069c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * convert scatterlist to SEC h/w link table format 10079c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * stop at cryptlen bytes 10089c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 100970bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipperstatic int sg_to_link_tbl(struct scatterlist *sg, int sg_count, 10109c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips int cryptlen, struct talitos_ptr *link_tbl_ptr) 10119c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 101270bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper int n_sg = sg_count; 101370bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper 101470bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper while (n_sg--) { 101581eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips to_talitos_ptr(link_tbl_ptr, sg_dma_address(sg)); 10169c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips link_tbl_ptr->len = cpu_to_be16(sg_dma_len(sg)); 10179c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips link_tbl_ptr->j_extent = 0; 10189c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips link_tbl_ptr++; 10199c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips cryptlen -= sg_dma_len(sg); 10204de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper sg = scatterwalk_sg_next(sg); 10219c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 10229c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 102370bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper /* adjust (decrease) last one (or two) entry's len to cryptlen */ 10249c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips link_tbl_ptr--; 1025c0e741d47859fcabb84a37589a4f49801ca8590aKim Phillips while (be16_to_cpu(link_tbl_ptr->len) <= (-cryptlen)) { 102670bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper /* Empty this entry, and move to previous one */ 102770bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper cryptlen += be16_to_cpu(link_tbl_ptr->len); 102870bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper link_tbl_ptr->len = 0; 102970bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper sg_count--; 103070bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper link_tbl_ptr--; 103170bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper } 10329c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips link_tbl_ptr->len = cpu_to_be16(be16_to_cpu(link_tbl_ptr->len) 10339c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips + cryptlen); 10349c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 10359c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* tag end of link table */ 10369c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN; 103770bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper 103870bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper return sg_count; 10399c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 10409c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 10419c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips/* 10429c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * fill in and submit ipsec_esp descriptor 10439c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 104456af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipperstatic int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, 10459c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips u8 *giv, u64 seq, 10469c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips void (*callback) (struct device *dev, 10479c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_desc *desc, 10489c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips void *context, int error)) 10499c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 10509c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct crypto_aead *aead = crypto_aead_reqtfm(areq); 10519c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_ctx *ctx = crypto_aead_ctx(aead); 10529c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct device *dev = ctx->dev; 10539c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_desc *desc = &edesc->desc; 10549c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unsigned int cryptlen = areq->cryptlen; 10559c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unsigned int authsize = ctx->authsize; 1056e41256f139b9148cfa12d2d057fec39e3d181ff0Kim Phillips unsigned int ivsize = crypto_aead_ivsize(aead); 1057fa86a26795b850cdf4e557898457a63e241c1aa1Kim Phillips int sg_count, ret; 1058fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips int sg_link_tbl_len; 10599c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 10609c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* hmac key */ 10619c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key, 10629c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 0, DMA_TO_DEVICE); 10639c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* hmac data */ 1064e41256f139b9148cfa12d2d057fec39e3d181ff0Kim Phillips map_single_talitos_ptr(dev, &desc->ptr[1], areq->assoclen + ivsize, 1065e41256f139b9148cfa12d2d057fec39e3d181ff0Kim Phillips sg_virt(areq->assoc), 0, DMA_TO_DEVICE); 10669c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* cipher iv */ 10679c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips map_single_talitos_ptr(dev, &desc->ptr[2], ivsize, giv ?: areq->iv, 0, 10689c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips DMA_TO_DEVICE); 10699c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 10709c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* cipher key */ 10719c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen, 10729c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips (char *)&ctx->key + ctx->authkeylen, 0, 10739c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips DMA_TO_DEVICE); 10749c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 10759c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* 10769c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * cipher in 10779c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * map and adjust cipher len to aead request cryptlen. 10789c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * extent is bytes of HMAC postpended to ciphertext, 10799c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * typically 12 for ipsec 10809c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 10819c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips desc->ptr[4].len = cpu_to_be16(cryptlen); 10829c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips desc->ptr[4].j_extent = authsize; 10839c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 1084e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1, 1085e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips (areq->src == areq->dst) ? DMA_BIDIRECTIONAL 1086e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips : DMA_TO_DEVICE, 10874de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper edesc->src_is_chained); 10889c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 10899c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (sg_count == 1) { 109081eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src)); 10919c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } else { 1092fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips sg_link_tbl_len = cryptlen; 1093fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips 1094962a9c99496f98041d14d64a9fdcf58050fefb4dKim Phillips if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV) 1095fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips sg_link_tbl_len = cryptlen + authsize; 1096e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips 1097fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips sg_count = sg_to_link_tbl(areq->src, sg_count, sg_link_tbl_len, 109870bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper &edesc->link_tbl[0]); 109970bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper if (sg_count > 1) { 110070bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP; 110181eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl); 1102e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips dma_sync_single_for_device(dev, edesc->dma_link_tbl, 1103e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips edesc->dma_len, 1104e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips DMA_BIDIRECTIONAL); 110570bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper } else { 110670bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper /* Only one segment now, so no link tbl needed */ 110781eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips to_talitos_ptr(&desc->ptr[4], 110881eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips sg_dma_address(areq->src)); 110970bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper } 11109c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 11119c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 11129c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* cipher out */ 11139c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips desc->ptr[5].len = cpu_to_be16(cryptlen); 11149c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips desc->ptr[5].j_extent = authsize; 11159c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 1116e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips if (areq->src != areq->dst) 11174de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper sg_count = talitos_map_sg(dev, areq->dst, 11184de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper edesc->dst_nents ? : 1, 11194de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper DMA_FROM_DEVICE, 11204de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper edesc->dst_is_chained); 11219c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 11229c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (sg_count == 1) { 112381eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst)); 11249c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } else { 11259c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_ptr *link_tbl_ptr = 1126f3c85bc1bc72b4cc8d58664a490a9d42bdb6565aLee Nipper &edesc->link_tbl[edesc->src_nents + 1]; 11279c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 112881eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl + 112981eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips (edesc->src_nents + 1) * 113081eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips sizeof(struct talitos_ptr)); 1131fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen, 1132fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips link_tbl_ptr); 1133fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips 1134f3c85bc1bc72b4cc8d58664a490a9d42bdb6565aLee Nipper /* Add an entry to the link table for ICV data */ 11359c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips link_tbl_ptr += sg_count - 1; 11369c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips link_tbl_ptr->j_extent = 0; 1137f3c85bc1bc72b4cc8d58664a490a9d42bdb6565aLee Nipper sg_count++; 11389c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips link_tbl_ptr++; 11399c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN; 11409c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips link_tbl_ptr->len = cpu_to_be16(authsize); 11419c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 11429c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* icv data follows link tables */ 114381eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips to_talitos_ptr(link_tbl_ptr, edesc->dma_link_tbl + 114481eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips (edesc->src_nents + edesc->dst_nents + 2) * 114581eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips sizeof(struct talitos_ptr)); 11469c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP; 11479c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl, 11489c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips edesc->dma_len, DMA_BIDIRECTIONAL); 11499c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 11509c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 11519c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* iv out */ 11529c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 0, 11539c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips DMA_FROM_DEVICE); 11549c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 11555228f0f79e983c2b39c202c75af901ceb0003fc1Kim Phillips ret = talitos_submit(dev, ctx->ch, desc, callback, areq); 1156fa86a26795b850cdf4e557898457a63e241c1aa1Kim Phillips if (ret != -EINPROGRESS) { 1157fa86a26795b850cdf4e557898457a63e241c1aa1Kim Phillips ipsec_esp_unmap(dev, edesc, areq); 1158fa86a26795b850cdf4e557898457a63e241c1aa1Kim Phillips kfree(edesc); 1159fa86a26795b850cdf4e557898457a63e241c1aa1Kim Phillips } 1160fa86a26795b850cdf4e557898457a63e241c1aa1Kim Phillips return ret; 11619c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 11629c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 11639c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips/* 11649c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * derive number of elements in scatterlist 11659c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 11664de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipperstatic int sg_count(struct scatterlist *sg_list, int nbytes, int *chained) 11679c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 11689c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct scatterlist *sg = sg_list; 11699c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips int sg_nents = 0; 11709c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 11714de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper *chained = 0; 11724de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper while (nbytes > 0) { 11739c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips sg_nents++; 11749c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips nbytes -= sg->length; 11754de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper if (!sg_is_last(sg) && (sg + 1)->length == 0) 11764de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper *chained = 1; 11774de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper sg = scatterwalk_sg_next(sg); 11789c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 11799c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 11809c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return sg_nents; 11819c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 11829c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 1183497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper/** 1184497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper * sg_copy_end_to_buffer - Copy end data from SG list to a linear buffer 1185497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper * @sgl: The SG list 1186497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper * @nents: Number of SG entries 1187497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper * @buf: Where to copy to 1188497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper * @buflen: The number of bytes to copy 1189497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper * @skip: The number of bytes to skip before copying. 1190497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper * Note: skip + buflen should equal SG total size. 1191497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper * 1192497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper * Returns the number of copied bytes. 1193497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper * 1194497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper **/ 1195497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipperstatic size_t sg_copy_end_to_buffer(struct scatterlist *sgl, unsigned int nents, 1196497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper void *buf, size_t buflen, unsigned int skip) 1197497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper{ 1198497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper unsigned int offset = 0; 1199497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper unsigned int boffset = 0; 1200497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct sg_mapping_iter miter; 1201497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper unsigned long flags; 1202497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper unsigned int sg_flags = SG_MITER_ATOMIC; 1203497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper size_t total_buffer = buflen + skip; 1204497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1205497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper sg_flags |= SG_MITER_FROM_SG; 1206497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1207497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper sg_miter_start(&miter, sgl, nents, sg_flags); 1208497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1209497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper local_irq_save(flags); 1210497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1211497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper while (sg_miter_next(&miter) && offset < total_buffer) { 1212497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper unsigned int len; 1213497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper unsigned int ignore; 1214497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1215497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper if ((offset + miter.length) > skip) { 1216497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper if (offset < skip) { 1217497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper /* Copy part of this segment */ 1218497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper ignore = skip - offset; 1219497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper len = miter.length - ignore; 12207260042b2d0397e7a8735ca47cd7839a5bb1210bLee Nipper if (boffset + len > buflen) 12217260042b2d0397e7a8735ca47cd7839a5bb1210bLee Nipper len = buflen - boffset; 1222497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper memcpy(buf + boffset, miter.addr + ignore, len); 1223497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper } else { 12247260042b2d0397e7a8735ca47cd7839a5bb1210bLee Nipper /* Copy all of this segment (up to buflen) */ 1225497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper len = miter.length; 12267260042b2d0397e7a8735ca47cd7839a5bb1210bLee Nipper if (boffset + len > buflen) 12277260042b2d0397e7a8735ca47cd7839a5bb1210bLee Nipper len = buflen - boffset; 1228497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper memcpy(buf + boffset, miter.addr, len); 1229497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper } 1230497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper boffset += len; 1231497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper } 1232497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper offset += miter.length; 1233497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper } 1234497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1235497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper sg_miter_stop(&miter); 1236497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1237497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper local_irq_restore(flags); 1238497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper return boffset; 1239497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper} 1240497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 12419c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips/* 124256af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper * allocate and map the extended descriptor 12439c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 12444de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipperstatic struct talitos_edesc *talitos_edesc_alloc(struct device *dev, 12454de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct scatterlist *src, 12464de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct scatterlist *dst, 1247497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper int hash_result, 12484de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper unsigned int cryptlen, 12494de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper unsigned int authsize, 12504de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper int icv_stashing, 12514de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper u32 cryptoflags) 12529c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 125356af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper struct talitos_edesc *edesc; 12549c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips int src_nents, dst_nents, alloc_len, dma_len; 12554de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper int src_chained, dst_chained = 0; 12564de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 1257586725f8604ef16ebbfdd66e73036e162ae00135Kim Phillips GFP_ATOMIC; 12589c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 12594de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper if (cryptlen + authsize > TALITOS_MAX_DATA_LEN) { 12604de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper dev_err(dev, "length exceeds h/w max limit\n"); 12619c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return ERR_PTR(-EINVAL); 12629c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 12639c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 12644de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper src_nents = sg_count(src, cryptlen + authsize, &src_chained); 12659c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips src_nents = (src_nents == 1) ? 0 : src_nents; 12669c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 1267497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper if (hash_result) { 1268497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper dst_nents = 0; 12699c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } else { 1270497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper if (dst == src) { 1271497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper dst_nents = src_nents; 1272497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper } else { 1273497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper dst_nents = sg_count(dst, cryptlen + authsize, 1274497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper &dst_chained); 1275497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper dst_nents = (dst_nents == 1) ? 0 : dst_nents; 1276497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper } 12779c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 12789c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 12799c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* 12809c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * allocate space for base edesc plus the link tables, 1281f3c85bc1bc72b4cc8d58664a490a9d42bdb6565aLee Nipper * allowing for two separate entries for ICV and generated ICV (+ 2), 12829c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * and the ICV data itself 12839c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 128456af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper alloc_len = sizeof(struct talitos_edesc); 12859c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (src_nents || dst_nents) { 1286f3c85bc1bc72b4cc8d58664a490a9d42bdb6565aLee Nipper dma_len = (src_nents + dst_nents + 2) * 12874de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper sizeof(struct talitos_ptr) + authsize; 12889c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips alloc_len += dma_len; 12899c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } else { 12909c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dma_len = 0; 12914de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper alloc_len += icv_stashing ? authsize : 0; 12929c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 12939c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 1294586725f8604ef16ebbfdd66e73036e162ae00135Kim Phillips edesc = kmalloc(alloc_len, GFP_DMA | flags); 12959c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (!edesc) { 12964de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper dev_err(dev, "could not allocate edescriptor\n"); 12979c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return ERR_PTR(-ENOMEM); 12989c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 12999c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 13009c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips edesc->src_nents = src_nents; 13019c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips edesc->dst_nents = dst_nents; 13024de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper edesc->src_is_chained = src_chained; 13034de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper edesc->dst_is_chained = dst_chained; 13049c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips edesc->dma_len = dma_len; 1305497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper if (dma_len) 1306497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0], 1307497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper edesc->dma_len, 1308497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper DMA_BIDIRECTIONAL); 13099c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 13109c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return edesc; 13119c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 13129c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 13134de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipperstatic struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, 13144de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper int icv_stashing) 13154de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper{ 13164de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct crypto_aead *authenc = crypto_aead_reqtfm(areq); 13174de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 13184de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 1319497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 0, 13204de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper areq->cryptlen, ctx->authsize, icv_stashing, 13214de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper areq->base.flags); 13224de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper} 13234de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 132456af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipperstatic int aead_encrypt(struct aead_request *req) 13259c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 13269c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct crypto_aead *authenc = crypto_aead_reqtfm(req); 13279c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 132856af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper struct talitos_edesc *edesc; 13299c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 13309c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* allocate extended descriptor */ 13314de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper edesc = aead_edesc_alloc(req, 0); 13329c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (IS_ERR(edesc)) 13339c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return PTR_ERR(edesc); 13349c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 13359c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* set encrypt */ 133670bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; 13379c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 13389c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_encrypt_done); 13399c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 13409c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 134156af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipperstatic int aead_decrypt(struct aead_request *req) 13429c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 13439c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct crypto_aead *authenc = crypto_aead_reqtfm(req); 13449c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 13459c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips unsigned int authsize = ctx->authsize; 1346fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips struct talitos_private *priv = dev_get_drvdata(ctx->dev); 134756af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper struct talitos_edesc *edesc; 13489c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct scatterlist *sg; 13499c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips void *icvdata; 13509c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 13519c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips req->cryptlen -= authsize; 13529c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 13539c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* allocate extended descriptor */ 13544de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper edesc = aead_edesc_alloc(req, 1); 13559c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (IS_ERR(edesc)) 13569c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return PTR_ERR(edesc); 13579c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 1358fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) && 1359e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips ((!edesc->src_nents && !edesc->dst_nents) || 1360e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) { 13619c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 1362fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips /* decrypt and check the ICV */ 1363e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips edesc->desc.hdr = ctx->desc_hdr_template | 1364e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips DESC_HDR_DIR_INBOUND | 1365fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips DESC_HDR_MODE1_MDEU_CICV; 13669c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 1367fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips /* reset integrity check result bits */ 1368fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips edesc->desc.hdr_lo = 0; 13699c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 1370e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips return ipsec_esp(edesc, req, NULL, 0, 1371e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips ipsec_esp_decrypt_hwauth_done); 1372fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips 1373e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips } 1374fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips 1375e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips /* Have to check the ICV with software */ 1376e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; 1377fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips 1378e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips /* stash incoming ICV for later cmp with ICV generated by the h/w */ 1379e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips if (edesc->dma_len) 1380e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips icvdata = &edesc->link_tbl[edesc->src_nents + 1381e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips edesc->dst_nents + 2]; 1382e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips else 1383e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips icvdata = &edesc->link_tbl[0]; 1384fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips 1385e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips sg = sg_last(req->src, edesc->src_nents ? : 1); 1386fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips 1387e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips memcpy(icvdata, (char *)sg_virt(sg) + sg->length - ctx->authsize, 1388e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips ctx->authsize); 1389fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips 1390e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_decrypt_swauth_done); 13919c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 13929c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 139356af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipperstatic int aead_givencrypt(struct aead_givcrypt_request *req) 13949c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 13959c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct aead_request *areq = &req->areq; 13969c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct crypto_aead *authenc = crypto_aead_reqtfm(areq); 13979c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 139856af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper struct talitos_edesc *edesc; 13999c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 14009c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* allocate extended descriptor */ 14014de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper edesc = aead_edesc_alloc(areq, 0); 14029c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (IS_ERR(edesc)) 14039c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return PTR_ERR(edesc); 14049c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 14059c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* set encrypt */ 140670bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; 14079c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 14089c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips memcpy(req->giv, ctx->iv, crypto_aead_ivsize(authenc)); 1409ba95487df97f9c45cfdae2a47de56235e7a13dc5Kim Phillips /* avoid consecutive packets going out with same IV */ 1410ba95487df97f9c45cfdae2a47de56235e7a13dc5Kim Phillips *(__be64 *)req->giv ^= cpu_to_be64(req->seq); 14119c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 14129c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return ipsec_esp(edesc, areq, req->giv, req->seq, 14139c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips ipsec_esp_encrypt_done); 14149c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 14159c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 14164de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipperstatic int ablkcipher_setkey(struct crypto_ablkcipher *cipher, 14174de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper const u8 *key, unsigned int keylen) 14184de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper{ 14194de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); 14204de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 14214de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper memcpy(&ctx->key, key, keylen); 14224de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper ctx->keylen = keylen; 14234de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 14244de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper return 0; 14254de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper} 14264de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 14274de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipperstatic void common_nonsnoop_unmap(struct device *dev, 14284de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct talitos_edesc *edesc, 14294de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct ablkcipher_request *areq) 14304de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper{ 14314de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); 14324de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE); 14334de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE); 14344de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 14354de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper talitos_sg_unmap(dev, edesc, areq->src, areq->dst); 14364de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 14374de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper if (edesc->dma_len) 14384de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, 14394de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper DMA_BIDIRECTIONAL); 14404de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper} 14414de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 14424de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipperstatic void ablkcipher_done(struct device *dev, 14434de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct talitos_desc *desc, void *context, 14444de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper int err) 14454de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper{ 14464de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct ablkcipher_request *areq = context; 144719bbbc635523703ece28409e59694d5b512b819eKim Phillips struct talitos_edesc *edesc; 144819bbbc635523703ece28409e59694d5b512b819eKim Phillips 144919bbbc635523703ece28409e59694d5b512b819eKim Phillips edesc = container_of(desc, struct talitos_edesc, desc); 14504de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 14514de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper common_nonsnoop_unmap(dev, edesc, areq); 14524de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 14534de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper kfree(edesc); 14544de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 14554de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper areq->base.complete(&areq->base, err); 14564de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper} 14574de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 14584de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipperstatic int common_nonsnoop(struct talitos_edesc *edesc, 14594de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct ablkcipher_request *areq, 14604de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper void (*callback) (struct device *dev, 14614de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct talitos_desc *desc, 14624de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper void *context, int error)) 14634de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper{ 14644de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); 14654de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); 14664de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct device *dev = ctx->dev; 14674de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct talitos_desc *desc = &edesc->desc; 14684de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper unsigned int cryptlen = areq->nbytes; 14694de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper unsigned int ivsize; 14704de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper int sg_count, ret; 14714de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 14724de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper /* first DWORD empty */ 14734de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper desc->ptr[0].len = 0; 147481eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips to_talitos_ptr(&desc->ptr[0], 0); 14754de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper desc->ptr[0].j_extent = 0; 14764de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 14774de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper /* cipher iv */ 14784de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper ivsize = crypto_ablkcipher_ivsize(cipher); 1479febec54286623a540cbe483a5d471ff6576a6e2eKim Phillips map_single_talitos_ptr(dev, &desc->ptr[1], ivsize, areq->info, 0, 14804de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper DMA_TO_DEVICE); 14814de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 14824de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper /* cipher key */ 14834de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen, 14844de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper (char *)&ctx->key, 0, DMA_TO_DEVICE); 14854de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 14864de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper /* 14874de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper * cipher in 14884de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper */ 14894de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper desc->ptr[3].len = cpu_to_be16(cryptlen); 14904de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper desc->ptr[3].j_extent = 0; 14914de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 14924de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1, 14934de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper (areq->src == areq->dst) ? DMA_BIDIRECTIONAL 14944de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper : DMA_TO_DEVICE, 14954de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper edesc->src_is_chained); 14964de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 14974de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper if (sg_count == 1) { 149881eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips to_talitos_ptr(&desc->ptr[3], sg_dma_address(areq->src)); 14994de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper } else { 15004de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper sg_count = sg_to_link_tbl(areq->src, sg_count, cryptlen, 15014de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper &edesc->link_tbl[0]); 15024de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper if (sg_count > 1) { 150381eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips to_talitos_ptr(&desc->ptr[3], edesc->dma_link_tbl); 15044de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper desc->ptr[3].j_extent |= DESC_PTR_LNKTBL_JUMP; 1505e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips dma_sync_single_for_device(dev, edesc->dma_link_tbl, 1506e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips edesc->dma_len, 1507e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips DMA_BIDIRECTIONAL); 15084de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper } else { 15094de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper /* Only one segment now, so no link tbl needed */ 151081eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips to_talitos_ptr(&desc->ptr[3], 151181eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips sg_dma_address(areq->src)); 15124de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper } 15134de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper } 15144de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 15154de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper /* cipher out */ 15164de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper desc->ptr[4].len = cpu_to_be16(cryptlen); 15174de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper desc->ptr[4].j_extent = 0; 15184de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 15194de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper if (areq->src != areq->dst) 15204de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper sg_count = talitos_map_sg(dev, areq->dst, 15214de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper edesc->dst_nents ? : 1, 15224de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper DMA_FROM_DEVICE, 15234de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper edesc->dst_is_chained); 15244de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 15254de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper if (sg_count == 1) { 152681eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->dst)); 15274de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper } else { 15284de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct talitos_ptr *link_tbl_ptr = 15294de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper &edesc->link_tbl[edesc->src_nents + 1]; 15304de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 153181eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl + 153281eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips (edesc->src_nents + 1) * 153381eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips sizeof(struct talitos_ptr)); 15344de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP; 15354de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen, 15364de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper link_tbl_ptr); 15374de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl, 15384de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper edesc->dma_len, DMA_BIDIRECTIONAL); 15394de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper } 15404de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 15414de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper /* iv out */ 15424de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv, 0, 15434de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper DMA_FROM_DEVICE); 15444de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 15454de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper /* last DWORD empty */ 15464de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper desc->ptr[6].len = 0; 154781eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips to_talitos_ptr(&desc->ptr[6], 0); 15484de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper desc->ptr[6].j_extent = 0; 15494de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 15505228f0f79e983c2b39c202c75af901ceb0003fc1Kim Phillips ret = talitos_submit(dev, ctx->ch, desc, callback, areq); 15514de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper if (ret != -EINPROGRESS) { 15524de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper common_nonsnoop_unmap(dev, edesc, areq); 15534de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper kfree(edesc); 15544de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper } 15554de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper return ret; 15564de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper} 15574de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 1558e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillipsstatic struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request * 1559e938e4656b3ee32e046ee8293411a07be9d72eb8Kim Phillips areq) 15604de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper{ 15614de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); 15624de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); 15634de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 1564497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 0, 1565497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper areq->nbytes, 0, 0, areq->base.flags); 15664de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper} 15674de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 15684de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipperstatic int ablkcipher_encrypt(struct ablkcipher_request *areq) 15694de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper{ 15704de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); 15714de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); 15724de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct talitos_edesc *edesc; 15734de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 15744de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper /* allocate extended descriptor */ 15754de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper edesc = ablkcipher_edesc_alloc(areq); 15764de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper if (IS_ERR(edesc)) 15774de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper return PTR_ERR(edesc); 15784de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 15794de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper /* set encrypt */ 15804de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; 15814de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 1582febec54286623a540cbe483a5d471ff6576a6e2eKim Phillips return common_nonsnoop(edesc, areq, ablkcipher_done); 15834de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper} 15844de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 15854de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipperstatic int ablkcipher_decrypt(struct ablkcipher_request *areq) 15864de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper{ 15874de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); 15884de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); 15894de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper struct talitos_edesc *edesc; 15904de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 15914de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper /* allocate extended descriptor */ 15924de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper edesc = ablkcipher_edesc_alloc(areq); 15934de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper if (IS_ERR(edesc)) 15944de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper return PTR_ERR(edesc); 15954de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 15964de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; 15974de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 1598febec54286623a540cbe483a5d471ff6576a6e2eKim Phillips return common_nonsnoop(edesc, areq, ablkcipher_done); 15994de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper} 16004de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper 1601497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipperstatic void common_nonsnoop_hash_unmap(struct device *dev, 1602497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct talitos_edesc *edesc, 1603497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct ahash_request *areq) 1604497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper{ 1605497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1606497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1607497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); 1608497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1609497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper /* When using hashctx-in, must unmap it. */ 1610497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper if (edesc->desc.ptr[1].len) 1611497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], 1612497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper DMA_TO_DEVICE); 1613497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1614497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper if (edesc->desc.ptr[2].len) 1615497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], 1616497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper DMA_TO_DEVICE); 1617497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1618497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL); 1619497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1620497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper if (edesc->dma_len) 1621497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, 1622497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper DMA_BIDIRECTIONAL); 1623497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1624497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper} 1625497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1626497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipperstatic void ahash_done(struct device *dev, 1627497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct talitos_desc *desc, void *context, 1628497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper int err) 1629497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper{ 1630497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct ahash_request *areq = context; 1631497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct talitos_edesc *edesc = 1632497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper container_of(desc, struct talitos_edesc, desc); 1633497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1634497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1635497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper if (!req_ctx->last && req_ctx->to_hash_later) { 1636497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper /* Position any partial block for next update/final/finup */ 1637497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later); 16385e833bc4166ea524c00a95e777e4344844ed189fLee Nipper req_ctx->nbuf = req_ctx->to_hash_later; 1639497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper } 1640497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper common_nonsnoop_hash_unmap(dev, edesc, areq); 1641497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1642497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper kfree(edesc); 1643497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1644497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper areq->base.complete(&areq->base, err); 1645497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper} 1646497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1647497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipperstatic int common_nonsnoop_hash(struct talitos_edesc *edesc, 1648497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct ahash_request *areq, unsigned int length, 1649497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper void (*callback) (struct device *dev, 1650497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct talitos_desc *desc, 1651497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper void *context, int error)) 1652497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper{ 1653497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1654497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1655497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1656497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct device *dev = ctx->dev; 1657497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct talitos_desc *desc = &edesc->desc; 1658497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper int sg_count, ret; 1659497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1660497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper /* first DWORD empty */ 1661497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper desc->ptr[0] = zero_entry; 1662497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 166360f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips /* hash context in */ 166460f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips if (!req_ctx->first || req_ctx->swinit) { 1665497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper map_single_talitos_ptr(dev, &desc->ptr[1], 1666497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper req_ctx->hw_context_size, 1667497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper (char *)req_ctx->hw_context, 0, 1668497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper DMA_TO_DEVICE); 166960f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips req_ctx->swinit = 0; 1670497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper } else { 1671497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper desc->ptr[1] = zero_entry; 1672497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper /* Indicate next op is not the first. */ 1673497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper req_ctx->first = 0; 1674497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper } 1675497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1676497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper /* HMAC key */ 1677497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper if (ctx->keylen) 1678497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen, 1679497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper (char *)&ctx->key, 0, DMA_TO_DEVICE); 1680497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper else 1681497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper desc->ptr[2] = zero_entry; 1682497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1683497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper /* 1684497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper * data in 1685497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper */ 1686497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper desc->ptr[3].len = cpu_to_be16(length); 1687497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper desc->ptr[3].j_extent = 0; 1688497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1689497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper sg_count = talitos_map_sg(dev, req_ctx->psrc, 1690497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper edesc->src_nents ? : 1, 1691497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper DMA_TO_DEVICE, 1692497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper edesc->src_is_chained); 1693497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1694497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper if (sg_count == 1) { 1695497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper to_talitos_ptr(&desc->ptr[3], sg_dma_address(req_ctx->psrc)); 1696497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper } else { 1697497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper sg_count = sg_to_link_tbl(req_ctx->psrc, sg_count, length, 1698497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper &edesc->link_tbl[0]); 1699497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper if (sg_count > 1) { 1700497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper desc->ptr[3].j_extent |= DESC_PTR_LNKTBL_JUMP; 1701497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper to_talitos_ptr(&desc->ptr[3], edesc->dma_link_tbl); 1702497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper dma_sync_single_for_device(ctx->dev, 1703497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper edesc->dma_link_tbl, 1704497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper edesc->dma_len, 1705497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper DMA_BIDIRECTIONAL); 1706497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper } else { 1707497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper /* Only one segment now, so no link tbl needed */ 1708497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper to_talitos_ptr(&desc->ptr[3], 1709497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper sg_dma_address(req_ctx->psrc)); 1710497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper } 1711497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper } 1712497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1713497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper /* fifth DWORD empty */ 1714497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper desc->ptr[4] = zero_entry; 1715497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1716497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper /* hash/HMAC out -or- hash context out */ 1717497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper if (req_ctx->last) 1718497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper map_single_talitos_ptr(dev, &desc->ptr[5], 1719497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper crypto_ahash_digestsize(tfm), 1720497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper areq->result, 0, DMA_FROM_DEVICE); 1721497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper else 1722497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper map_single_talitos_ptr(dev, &desc->ptr[5], 1723497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper req_ctx->hw_context_size, 1724497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper req_ctx->hw_context, 0, DMA_FROM_DEVICE); 1725497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1726497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper /* last DWORD empty */ 1727497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper desc->ptr[6] = zero_entry; 1728497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 17295228f0f79e983c2b39c202c75af901ceb0003fc1Kim Phillips ret = talitos_submit(dev, ctx->ch, desc, callback, areq); 1730497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper if (ret != -EINPROGRESS) { 1731497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper common_nonsnoop_hash_unmap(dev, edesc, areq); 1732497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper kfree(edesc); 1733497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper } 1734497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper return ret; 1735497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper} 1736497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1737497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipperstatic struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq, 1738497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper unsigned int nbytes) 1739497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper{ 1740497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1741497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1742497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1743497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1744497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, 1, 1745497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper nbytes, 0, 0, areq->base.flags); 1746497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper} 1747497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1748497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipperstatic int ahash_init(struct ahash_request *areq) 1749497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper{ 1750497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1751497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1752497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1753497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper /* Initialize the context */ 17545e833bc4166ea524c00a95e777e4344844ed189fLee Nipper req_ctx->nbuf = 0; 175560f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips req_ctx->first = 1; /* first indicates h/w must init its context */ 175660f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips req_ctx->swinit = 0; /* assume h/w init of context */ 1757497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper req_ctx->hw_context_size = 1758497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) 1759497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 1760497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; 1761497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1762497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper return 0; 1763497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper} 1764497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 176560f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips/* 176660f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips * on h/w without explicit sha224 support, we initialize h/w context 176760f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips * manually with sha224 constants, and tell it to run sha256. 176860f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips */ 176960f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillipsstatic int ahash_init_sha224_swinit(struct ahash_request *areq) 177060f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips{ 177160f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 177260f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips 177360f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips ahash_init(areq); 177460f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/ 177560f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips 1776a752447af5b61f19e9c50322d9b07cea9a086084Kim Phillips req_ctx->hw_context[0] = SHA224_H0; 1777a752447af5b61f19e9c50322d9b07cea9a086084Kim Phillips req_ctx->hw_context[1] = SHA224_H1; 1778a752447af5b61f19e9c50322d9b07cea9a086084Kim Phillips req_ctx->hw_context[2] = SHA224_H2; 1779a752447af5b61f19e9c50322d9b07cea9a086084Kim Phillips req_ctx->hw_context[3] = SHA224_H3; 1780a752447af5b61f19e9c50322d9b07cea9a086084Kim Phillips req_ctx->hw_context[4] = SHA224_H4; 1781a752447af5b61f19e9c50322d9b07cea9a086084Kim Phillips req_ctx->hw_context[5] = SHA224_H5; 1782a752447af5b61f19e9c50322d9b07cea9a086084Kim Phillips req_ctx->hw_context[6] = SHA224_H6; 1783a752447af5b61f19e9c50322d9b07cea9a086084Kim Phillips req_ctx->hw_context[7] = SHA224_H7; 178460f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips 178560f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips /* init 64-bit count */ 178660f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips req_ctx->hw_context[8] = 0; 178760f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips req_ctx->hw_context[9] = 0; 178860f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips 178960f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips return 0; 179060f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips} 179160f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips 1792497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipperstatic int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) 1793497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper{ 1794497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1795497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1796497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1797497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct talitos_edesc *edesc; 1798497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper unsigned int blocksize = 1799497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 1800497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper unsigned int nbytes_to_hash; 1801497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper unsigned int to_hash_later; 18025e833bc4166ea524c00a95e777e4344844ed189fLee Nipper unsigned int nsg; 1803497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper int chained; 1804497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 18055e833bc4166ea524c00a95e777e4344844ed189fLee Nipper if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) { 18065e833bc4166ea524c00a95e777e4344844ed189fLee Nipper /* Buffer up to one whole block */ 1807497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper sg_copy_to_buffer(areq->src, 1808497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper sg_count(areq->src, nbytes, &chained), 18095e833bc4166ea524c00a95e777e4344844ed189fLee Nipper req_ctx->buf + req_ctx->nbuf, nbytes); 18105e833bc4166ea524c00a95e777e4344844ed189fLee Nipper req_ctx->nbuf += nbytes; 1811497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper return 0; 1812497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper } 1813497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 18145e833bc4166ea524c00a95e777e4344844ed189fLee Nipper /* At least (blocksize + 1) bytes are available to hash */ 18155e833bc4166ea524c00a95e777e4344844ed189fLee Nipper nbytes_to_hash = nbytes + req_ctx->nbuf; 18165e833bc4166ea524c00a95e777e4344844ed189fLee Nipper to_hash_later = nbytes_to_hash & (blocksize - 1); 18175e833bc4166ea524c00a95e777e4344844ed189fLee Nipper 18185e833bc4166ea524c00a95e777e4344844ed189fLee Nipper if (req_ctx->last) 18195e833bc4166ea524c00a95e777e4344844ed189fLee Nipper to_hash_later = 0; 18205e833bc4166ea524c00a95e777e4344844ed189fLee Nipper else if (to_hash_later) 18215e833bc4166ea524c00a95e777e4344844ed189fLee Nipper /* There is a partial block. Hash the full block(s) now */ 18225e833bc4166ea524c00a95e777e4344844ed189fLee Nipper nbytes_to_hash -= to_hash_later; 18235e833bc4166ea524c00a95e777e4344844ed189fLee Nipper else { 18245e833bc4166ea524c00a95e777e4344844ed189fLee Nipper /* Keep one block buffered */ 18255e833bc4166ea524c00a95e777e4344844ed189fLee Nipper nbytes_to_hash -= blocksize; 18265e833bc4166ea524c00a95e777e4344844ed189fLee Nipper to_hash_later = blocksize; 18275e833bc4166ea524c00a95e777e4344844ed189fLee Nipper } 18285e833bc4166ea524c00a95e777e4344844ed189fLee Nipper 18295e833bc4166ea524c00a95e777e4344844ed189fLee Nipper /* Chain in any previously buffered data */ 18305e833bc4166ea524c00a95e777e4344844ed189fLee Nipper if (req_ctx->nbuf) { 18315e833bc4166ea524c00a95e777e4344844ed189fLee Nipper nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1; 18325e833bc4166ea524c00a95e777e4344844ed189fLee Nipper sg_init_table(req_ctx->bufsl, nsg); 18335e833bc4166ea524c00a95e777e4344844ed189fLee Nipper sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf); 18345e833bc4166ea524c00a95e777e4344844ed189fLee Nipper if (nsg > 1) 18355e833bc4166ea524c00a95e777e4344844ed189fLee Nipper scatterwalk_sg_chain(req_ctx->bufsl, 2, areq->src); 1836497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper req_ctx->psrc = req_ctx->bufsl; 18375e833bc4166ea524c00a95e777e4344844ed189fLee Nipper } else 1838497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper req_ctx->psrc = areq->src; 18395e833bc4166ea524c00a95e777e4344844ed189fLee Nipper 18405e833bc4166ea524c00a95e777e4344844ed189fLee Nipper if (to_hash_later) { 18415e833bc4166ea524c00a95e777e4344844ed189fLee Nipper int nents = sg_count(areq->src, nbytes, &chained); 18425e833bc4166ea524c00a95e777e4344844ed189fLee Nipper sg_copy_end_to_buffer(areq->src, nents, 18435e833bc4166ea524c00a95e777e4344844ed189fLee Nipper req_ctx->bufnext, 18445e833bc4166ea524c00a95e777e4344844ed189fLee Nipper to_hash_later, 18455e833bc4166ea524c00a95e777e4344844ed189fLee Nipper nbytes - to_hash_later); 1846497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper } 18475e833bc4166ea524c00a95e777e4344844ed189fLee Nipper req_ctx->to_hash_later = to_hash_later; 1848497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 18495e833bc4166ea524c00a95e777e4344844ed189fLee Nipper /* Allocate extended descriptor */ 1850497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper edesc = ahash_edesc_alloc(areq, nbytes_to_hash); 1851497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper if (IS_ERR(edesc)) 1852497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper return PTR_ERR(edesc); 1853497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1854497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper edesc->desc.hdr = ctx->desc_hdr_template; 1855497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1856497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper /* On last one, request SEC to pad; otherwise continue */ 1857497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper if (req_ctx->last) 1858497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD; 1859497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper else 1860497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT; 1861497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 186260f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips /* request SEC to INIT hash. */ 186360f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips if (req_ctx->first && !req_ctx->swinit) 1864497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT; 1865497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1866497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper /* When the tfm context has a keylen, it's an HMAC. 1867497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper * A first or last (ie. not middle) descriptor must request HMAC. 1868497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper */ 1869497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper if (ctx->keylen && (req_ctx->first || req_ctx->last)) 1870497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC; 1871497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1872497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, 1873497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper ahash_done); 1874497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper} 1875497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1876497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipperstatic int ahash_update(struct ahash_request *areq) 1877497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper{ 1878497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1879497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1880497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper req_ctx->last = 0; 1881497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1882497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper return ahash_process_req(areq, areq->nbytes); 1883497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper} 1884497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1885497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipperstatic int ahash_final(struct ahash_request *areq) 1886497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper{ 1887497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1888497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1889497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper req_ctx->last = 1; 1890497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1891497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper return ahash_process_req(areq, 0); 1892497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper} 1893497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1894497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipperstatic int ahash_finup(struct ahash_request *areq) 1895497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper{ 1896497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1897497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1898497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper req_ctx->last = 1; 1899497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1900497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper return ahash_process_req(areq, areq->nbytes); 1901497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper} 1902497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1903497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipperstatic int ahash_digest(struct ahash_request *areq) 1904497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper{ 1905497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 190660f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 1907497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 190860f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips ahash->init(areq); 1909497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper req_ctx->last = 1; 1910497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 1911497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper return ahash_process_req(areq, areq->nbytes); 1912497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper} 1913497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 191479b3a418e090248d00ceba40b81da9dfac753367Lee Nipperstruct keyhash_result { 191579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper struct completion completion; 191679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper int err; 191779b3a418e090248d00ceba40b81da9dfac753367Lee Nipper}; 191879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper 191979b3a418e090248d00ceba40b81da9dfac753367Lee Nipperstatic void keyhash_complete(struct crypto_async_request *req, int err) 192079b3a418e090248d00ceba40b81da9dfac753367Lee Nipper{ 192179b3a418e090248d00ceba40b81da9dfac753367Lee Nipper struct keyhash_result *res = req->data; 192279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper 192379b3a418e090248d00ceba40b81da9dfac753367Lee Nipper if (err == -EINPROGRESS) 192479b3a418e090248d00ceba40b81da9dfac753367Lee Nipper return; 192579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper 192679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper res->err = err; 192779b3a418e090248d00ceba40b81da9dfac753367Lee Nipper complete(&res->completion); 192879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper} 192979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper 193079b3a418e090248d00ceba40b81da9dfac753367Lee Nipperstatic int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen, 193179b3a418e090248d00ceba40b81da9dfac753367Lee Nipper u8 *hash) 193279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper{ 193379b3a418e090248d00ceba40b81da9dfac753367Lee Nipper struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 193479b3a418e090248d00ceba40b81da9dfac753367Lee Nipper 193579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper struct scatterlist sg[1]; 193679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper struct ahash_request *req; 193779b3a418e090248d00ceba40b81da9dfac753367Lee Nipper struct keyhash_result hresult; 193879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper int ret; 193979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper 194079b3a418e090248d00ceba40b81da9dfac753367Lee Nipper init_completion(&hresult.completion); 194179b3a418e090248d00ceba40b81da9dfac753367Lee Nipper 194279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper req = ahash_request_alloc(tfm, GFP_KERNEL); 194379b3a418e090248d00ceba40b81da9dfac753367Lee Nipper if (!req) 194479b3a418e090248d00ceba40b81da9dfac753367Lee Nipper return -ENOMEM; 194579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper 194679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper /* Keep tfm keylen == 0 during hash of the long key */ 194779b3a418e090248d00ceba40b81da9dfac753367Lee Nipper ctx->keylen = 0; 194879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 194979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper keyhash_complete, &hresult); 195079b3a418e090248d00ceba40b81da9dfac753367Lee Nipper 195179b3a418e090248d00ceba40b81da9dfac753367Lee Nipper sg_init_one(&sg[0], key, keylen); 195279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper 195379b3a418e090248d00ceba40b81da9dfac753367Lee Nipper ahash_request_set_crypt(req, sg, hash, keylen); 195479b3a418e090248d00ceba40b81da9dfac753367Lee Nipper ret = crypto_ahash_digest(req); 195579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper switch (ret) { 195679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper case 0: 195779b3a418e090248d00ceba40b81da9dfac753367Lee Nipper break; 195879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper case -EINPROGRESS: 195979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper case -EBUSY: 196079b3a418e090248d00ceba40b81da9dfac753367Lee Nipper ret = wait_for_completion_interruptible( 196179b3a418e090248d00ceba40b81da9dfac753367Lee Nipper &hresult.completion); 196279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper if (!ret) 196379b3a418e090248d00ceba40b81da9dfac753367Lee Nipper ret = hresult.err; 196479b3a418e090248d00ceba40b81da9dfac753367Lee Nipper break; 196579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper default: 196679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper break; 196779b3a418e090248d00ceba40b81da9dfac753367Lee Nipper } 196879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper ahash_request_free(req); 196979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper 197079b3a418e090248d00ceba40b81da9dfac753367Lee Nipper return ret; 197179b3a418e090248d00ceba40b81da9dfac753367Lee Nipper} 197279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper 197379b3a418e090248d00ceba40b81da9dfac753367Lee Nipperstatic int ahash_setkey(struct crypto_ahash *tfm, const u8 *key, 197479b3a418e090248d00ceba40b81da9dfac753367Lee Nipper unsigned int keylen) 197579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper{ 197679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 197779b3a418e090248d00ceba40b81da9dfac753367Lee Nipper unsigned int blocksize = 197879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 197979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper unsigned int digestsize = crypto_ahash_digestsize(tfm); 198079b3a418e090248d00ceba40b81da9dfac753367Lee Nipper unsigned int keysize = keylen; 198179b3a418e090248d00ceba40b81da9dfac753367Lee Nipper u8 hash[SHA512_DIGEST_SIZE]; 198279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper int ret; 198379b3a418e090248d00ceba40b81da9dfac753367Lee Nipper 198479b3a418e090248d00ceba40b81da9dfac753367Lee Nipper if (keylen <= blocksize) 198579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper memcpy(ctx->key, key, keysize); 198679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper else { 198779b3a418e090248d00ceba40b81da9dfac753367Lee Nipper /* Must get the hash of the long key */ 198879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper ret = keyhash(tfm, key, keylen, hash); 198979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper 199079b3a418e090248d00ceba40b81da9dfac753367Lee Nipper if (ret) { 199179b3a418e090248d00ceba40b81da9dfac753367Lee Nipper crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 199279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper return -EINVAL; 199379b3a418e090248d00ceba40b81da9dfac753367Lee Nipper } 199479b3a418e090248d00ceba40b81da9dfac753367Lee Nipper 199579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper keysize = digestsize; 199679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper memcpy(ctx->key, hash, digestsize); 199779b3a418e090248d00ceba40b81da9dfac753367Lee Nipper } 199879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper 199979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper ctx->keylen = keysize; 200079b3a418e090248d00ceba40b81da9dfac753367Lee Nipper 200179b3a418e090248d00ceba40b81da9dfac753367Lee Nipper return 0; 200279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper} 200379b3a418e090248d00ceba40b81da9dfac753367Lee Nipper 200479b3a418e090248d00ceba40b81da9dfac753367Lee Nipper 20059c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstruct talitos_alg_template { 2006d5e4aaefd99797929e4455db48d8f9a8728f05f4Lee Nipper u32 type; 2007d5e4aaefd99797929e4455db48d8f9a8728f05f4Lee Nipper union { 2008d5e4aaefd99797929e4455db48d8f9a8728f05f4Lee Nipper struct crypto_alg crypto; 2009acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper struct ahash_alg hash; 2010d5e4aaefd99797929e4455db48d8f9a8728f05f4Lee Nipper } alg; 20119c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips __be32 desc_hdr_template; 20129c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips}; 20139c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 20149c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstatic struct talitos_alg_template driver_algs[] = { 201556af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */ 2016d5e4aaefd99797929e4455db48d8f9a8728f05f4Lee Nipper { .type = CRYPTO_ALG_TYPE_AEAD, 2017d5e4aaefd99797929e4455db48d8f9a8728f05f4Lee Nipper .alg.crypto = { 201856af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_name = "authenc(hmac(sha1),cbc(aes))", 201956af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_driver_name = "authenc-hmac-sha1-cbc-aes-talitos", 202056af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_blocksize = AES_BLOCK_SIZE, 202156af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, 202256af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_type = &crypto_aead_type, 202356af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_aead = { 202456af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .setkey = aead_setkey, 202556af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .setauthsize = aead_setauthsize, 202656af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .encrypt = aead_encrypt, 202756af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .decrypt = aead_decrypt, 202856af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .givencrypt = aead_givencrypt, 202956af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .geniv = "<built-in>", 203056af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .ivsize = AES_BLOCK_SIZE, 203156af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .maxauthsize = SHA1_DIGEST_SIZE, 203256af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper } 203356af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper }, 20349c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 20359c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips DESC_HDR_SEL0_AESU | 20369c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips DESC_HDR_MODE0_AESU_CBC | 20379c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips DESC_HDR_SEL1_MDEUA | 20389c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips DESC_HDR_MODE1_MDEU_INIT | 20399c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips DESC_HDR_MODE1_MDEU_PAD | 20409c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips DESC_HDR_MODE1_MDEU_SHA1_HMAC, 204170bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper }, 2042d5e4aaefd99797929e4455db48d8f9a8728f05f4Lee Nipper { .type = CRYPTO_ALG_TYPE_AEAD, 2043d5e4aaefd99797929e4455db48d8f9a8728f05f4Lee Nipper .alg.crypto = { 204456af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", 204556af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_driver_name = "authenc-hmac-sha1-cbc-3des-talitos", 204656af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_blocksize = DES3_EDE_BLOCK_SIZE, 204756af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, 204856af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_type = &crypto_aead_type, 204956af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_aead = { 205056af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .setkey = aead_setkey, 205156af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .setauthsize = aead_setauthsize, 205256af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .encrypt = aead_encrypt, 205356af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .decrypt = aead_decrypt, 205456af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .givencrypt = aead_givencrypt, 205556af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .geniv = "<built-in>", 205656af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .ivsize = DES3_EDE_BLOCK_SIZE, 205756af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .maxauthsize = SHA1_DIGEST_SIZE, 205856af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper } 205956af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper }, 206070bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 206170bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper DESC_HDR_SEL0_DEU | 206270bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper DESC_HDR_MODE0_DEU_CBC | 206370bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper DESC_HDR_MODE0_DEU_3DES | 206470bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper DESC_HDR_SEL1_MDEUA | 206570bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper DESC_HDR_MODE1_MDEU_INIT | 206670bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper DESC_HDR_MODE1_MDEU_PAD | 206770bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper DESC_HDR_MODE1_MDEU_SHA1_HMAC, 20683952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper }, 2069d5e4aaefd99797929e4455db48d8f9a8728f05f4Lee Nipper { .type = CRYPTO_ALG_TYPE_AEAD, 2070d5e4aaefd99797929e4455db48d8f9a8728f05f4Lee Nipper .alg.crypto = { 207156af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_name = "authenc(hmac(sha256),cbc(aes))", 207256af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_driver_name = "authenc-hmac-sha256-cbc-aes-talitos", 207356af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_blocksize = AES_BLOCK_SIZE, 207456af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, 207556af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_type = &crypto_aead_type, 207656af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_aead = { 207756af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .setkey = aead_setkey, 207856af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .setauthsize = aead_setauthsize, 207956af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .encrypt = aead_encrypt, 208056af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .decrypt = aead_decrypt, 208156af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .givencrypt = aead_givencrypt, 208256af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .geniv = "<built-in>", 208356af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .ivsize = AES_BLOCK_SIZE, 208456af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .maxauthsize = SHA256_DIGEST_SIZE, 208556af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper } 208656af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper }, 20873952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 20883952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_SEL0_AESU | 20893952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_MODE0_AESU_CBC | 20903952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_SEL1_MDEUA | 20913952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_MODE1_MDEU_INIT | 20923952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_MODE1_MDEU_PAD | 20933952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_MODE1_MDEU_SHA256_HMAC, 20943952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper }, 2095d5e4aaefd99797929e4455db48d8f9a8728f05f4Lee Nipper { .type = CRYPTO_ALG_TYPE_AEAD, 2096d5e4aaefd99797929e4455db48d8f9a8728f05f4Lee Nipper .alg.crypto = { 209756af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_name = "authenc(hmac(sha256),cbc(des3_ede))", 209856af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_driver_name = "authenc-hmac-sha256-cbc-3des-talitos", 209956af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_blocksize = DES3_EDE_BLOCK_SIZE, 210056af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, 210156af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_type = &crypto_aead_type, 210256af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_aead = { 210356af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .setkey = aead_setkey, 210456af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .setauthsize = aead_setauthsize, 210556af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .encrypt = aead_encrypt, 210656af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .decrypt = aead_decrypt, 210756af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .givencrypt = aead_givencrypt, 210856af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .geniv = "<built-in>", 210956af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .ivsize = DES3_EDE_BLOCK_SIZE, 211056af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .maxauthsize = SHA256_DIGEST_SIZE, 211156af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper } 211256af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper }, 21133952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 21143952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_SEL0_DEU | 21153952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_MODE0_DEU_CBC | 21163952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_MODE0_DEU_3DES | 21173952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_SEL1_MDEUA | 21183952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_MODE1_MDEU_INIT | 21193952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_MODE1_MDEU_PAD | 21203952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_MODE1_MDEU_SHA256_HMAC, 21213952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper }, 2122d5e4aaefd99797929e4455db48d8f9a8728f05f4Lee Nipper { .type = CRYPTO_ALG_TYPE_AEAD, 2123d5e4aaefd99797929e4455db48d8f9a8728f05f4Lee Nipper .alg.crypto = { 212456af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_name = "authenc(hmac(md5),cbc(aes))", 212556af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_driver_name = "authenc-hmac-md5-cbc-aes-talitos", 212656af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_blocksize = AES_BLOCK_SIZE, 212756af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, 212856af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_type = &crypto_aead_type, 212956af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_aead = { 213056af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .setkey = aead_setkey, 213156af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .setauthsize = aead_setauthsize, 213256af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .encrypt = aead_encrypt, 213356af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .decrypt = aead_decrypt, 213456af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .givencrypt = aead_givencrypt, 213556af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .geniv = "<built-in>", 213656af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .ivsize = AES_BLOCK_SIZE, 213756af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .maxauthsize = MD5_DIGEST_SIZE, 213856af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper } 213956af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper }, 21403952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 21413952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_SEL0_AESU | 21423952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_MODE0_AESU_CBC | 21433952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_SEL1_MDEUA | 21443952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_MODE1_MDEU_INIT | 21453952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_MODE1_MDEU_PAD | 21463952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_MODE1_MDEU_MD5_HMAC, 21473952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper }, 2148d5e4aaefd99797929e4455db48d8f9a8728f05f4Lee Nipper { .type = CRYPTO_ALG_TYPE_AEAD, 2149d5e4aaefd99797929e4455db48d8f9a8728f05f4Lee Nipper .alg.crypto = { 215056af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 215156af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_driver_name = "authenc-hmac-md5-cbc-3des-talitos", 215256af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_blocksize = DES3_EDE_BLOCK_SIZE, 215356af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, 215456af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_type = &crypto_aead_type, 215556af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .cra_aead = { 215656af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .setkey = aead_setkey, 215756af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .setauthsize = aead_setauthsize, 215856af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .encrypt = aead_encrypt, 215956af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .decrypt = aead_decrypt, 216056af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .givencrypt = aead_givencrypt, 216156af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .geniv = "<built-in>", 216256af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .ivsize = DES3_EDE_BLOCK_SIZE, 216356af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper .maxauthsize = MD5_DIGEST_SIZE, 216456af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper } 216556af8cd44b05bd9649103b76a6e1e575682990e4Lee Nipper }, 21663952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 21673952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_SEL0_DEU | 21683952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_MODE0_DEU_CBC | 21693952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_MODE0_DEU_3DES | 21703952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_SEL1_MDEUA | 21713952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_MODE1_MDEU_INIT | 21723952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_MODE1_MDEU_PAD | 21733952f17ed63434cc2154c3765ff97e1d4adab042Lee Nipper DESC_HDR_MODE1_MDEU_MD5_HMAC, 21744de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper }, 21754de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper /* ABLKCIPHER algorithms. */ 2176d5e4aaefd99797929e4455db48d8f9a8728f05f4Lee Nipper { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2177d5e4aaefd99797929e4455db48d8f9a8728f05f4Lee Nipper .alg.crypto = { 21784de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .cra_name = "cbc(aes)", 21794de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .cra_driver_name = "cbc-aes-talitos", 21804de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .cra_blocksize = AES_BLOCK_SIZE, 21814de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 21824de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper CRYPTO_ALG_ASYNC, 21834de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .cra_type = &crypto_ablkcipher_type, 21844de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .cra_ablkcipher = { 21854de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .setkey = ablkcipher_setkey, 21864de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .encrypt = ablkcipher_encrypt, 21874de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .decrypt = ablkcipher_decrypt, 21884de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .geniv = "eseqiv", 21894de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .min_keysize = AES_MIN_KEY_SIZE, 21904de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .max_keysize = AES_MAX_KEY_SIZE, 21914de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .ivsize = AES_BLOCK_SIZE, 21924de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper } 21934de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper }, 21944de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 21954de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper DESC_HDR_SEL0_AESU | 21964de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper DESC_HDR_MODE0_AESU_CBC, 21974de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper }, 2198d5e4aaefd99797929e4455db48d8f9a8728f05f4Lee Nipper { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2199d5e4aaefd99797929e4455db48d8f9a8728f05f4Lee Nipper .alg.crypto = { 22004de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .cra_name = "cbc(des3_ede)", 22014de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .cra_driver_name = "cbc-3des-talitos", 22024de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .cra_blocksize = DES3_EDE_BLOCK_SIZE, 22034de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 22044de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper CRYPTO_ALG_ASYNC, 22054de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .cra_type = &crypto_ablkcipher_type, 22064de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .cra_ablkcipher = { 22074de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .setkey = ablkcipher_setkey, 22084de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .encrypt = ablkcipher_encrypt, 22094de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .decrypt = ablkcipher_decrypt, 22104de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .geniv = "eseqiv", 22114de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .min_keysize = DES3_EDE_KEY_SIZE, 22124de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .max_keysize = DES3_EDE_KEY_SIZE, 22134de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .ivsize = DES3_EDE_BLOCK_SIZE, 22144de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper } 22154de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper }, 22164de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 22174de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper DESC_HDR_SEL0_DEU | 22184de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper DESC_HDR_MODE0_DEU_CBC | 22194de9d0b547b97e40c93a885ac6246c2c5fef05cbLee Nipper DESC_HDR_MODE0_DEU_3DES, 2220497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper }, 2221497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper /* AHASH algorithms. */ 2222497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper { .type = CRYPTO_ALG_TYPE_AHASH, 2223497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .alg.hash = { 2224497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .init = ahash_init, 2225497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .update = ahash_update, 2226497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .final = ahash_final, 2227497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .finup = ahash_finup, 2228497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .digest = ahash_digest, 2229497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .halg.digestsize = MD5_DIGEST_SIZE, 2230497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .halg.base = { 2231497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_name = "md5", 2232497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_driver_name = "md5-talitos", 2233497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_blocksize = MD5_BLOCK_SIZE, 2234497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2235497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper CRYPTO_ALG_ASYNC, 2236497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_type = &crypto_ahash_type 2237497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper } 2238497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper }, 2239497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2240497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper DESC_HDR_SEL0_MDEUA | 2241497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper DESC_HDR_MODE0_MDEU_MD5, 2242497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper }, 2243497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper { .type = CRYPTO_ALG_TYPE_AHASH, 2244497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .alg.hash = { 2245497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .init = ahash_init, 2246497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .update = ahash_update, 2247497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .final = ahash_final, 2248497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .finup = ahash_finup, 2249497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .digest = ahash_digest, 2250497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .halg.digestsize = SHA1_DIGEST_SIZE, 2251497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .halg.base = { 2252497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_name = "sha1", 2253497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_driver_name = "sha1-talitos", 2254497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_blocksize = SHA1_BLOCK_SIZE, 2255497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2256497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper CRYPTO_ALG_ASYNC, 2257497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_type = &crypto_ahash_type 2258497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper } 2259497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper }, 2260497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2261497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper DESC_HDR_SEL0_MDEUA | 2262497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper DESC_HDR_MODE0_MDEU_SHA1, 2263497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper }, 2264497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper { .type = CRYPTO_ALG_TYPE_AHASH, 2265497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .alg.hash = { 2266497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .init = ahash_init, 2267497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .update = ahash_update, 2268497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .final = ahash_final, 2269497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .finup = ahash_finup, 2270497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .digest = ahash_digest, 227160f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips .halg.digestsize = SHA224_DIGEST_SIZE, 227260f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips .halg.base = { 227360f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips .cra_name = "sha224", 227460f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips .cra_driver_name = "sha224-talitos", 227560f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips .cra_blocksize = SHA224_BLOCK_SIZE, 227660f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips .cra_flags = CRYPTO_ALG_TYPE_AHASH | 227760f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips CRYPTO_ALG_ASYNC, 227860f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips .cra_type = &crypto_ahash_type 227960f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips } 228060f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips }, 228160f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 228260f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips DESC_HDR_SEL0_MDEUA | 228360f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips DESC_HDR_MODE0_MDEU_SHA224, 228460f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips }, 228560f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips { .type = CRYPTO_ALG_TYPE_AHASH, 228660f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips .alg.hash = { 228760f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips .init = ahash_init, 228860f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips .update = ahash_update, 228960f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips .final = ahash_final, 229060f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips .finup = ahash_finup, 229160f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips .digest = ahash_digest, 2292497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .halg.digestsize = SHA256_DIGEST_SIZE, 2293497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .halg.base = { 2294497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_name = "sha256", 2295497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_driver_name = "sha256-talitos", 2296497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_blocksize = SHA256_BLOCK_SIZE, 2297497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2298497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper CRYPTO_ALG_ASYNC, 2299497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_type = &crypto_ahash_type 2300497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper } 2301497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper }, 2302497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2303497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper DESC_HDR_SEL0_MDEUA | 2304497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper DESC_HDR_MODE0_MDEU_SHA256, 2305497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper }, 2306497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper { .type = CRYPTO_ALG_TYPE_AHASH, 2307497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .alg.hash = { 2308497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .init = ahash_init, 2309497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .update = ahash_update, 2310497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .final = ahash_final, 2311497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .finup = ahash_finup, 2312497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .digest = ahash_digest, 2313497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .halg.digestsize = SHA384_DIGEST_SIZE, 2314497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .halg.base = { 2315497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_name = "sha384", 2316497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_driver_name = "sha384-talitos", 2317497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_blocksize = SHA384_BLOCK_SIZE, 2318497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2319497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper CRYPTO_ALG_ASYNC, 2320497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_type = &crypto_ahash_type 2321497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper } 2322497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper }, 2323497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2324497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper DESC_HDR_SEL0_MDEUB | 2325497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper DESC_HDR_MODE0_MDEUB_SHA384, 2326497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper }, 2327497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper { .type = CRYPTO_ALG_TYPE_AHASH, 2328497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .alg.hash = { 2329497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .init = ahash_init, 2330497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .update = ahash_update, 2331497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .final = ahash_final, 2332497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .finup = ahash_finup, 2333497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .digest = ahash_digest, 2334497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .halg.digestsize = SHA512_DIGEST_SIZE, 2335497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .halg.base = { 2336497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_name = "sha512", 2337497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_driver_name = "sha512-talitos", 2338497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_blocksize = SHA512_BLOCK_SIZE, 2339497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2340497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper CRYPTO_ALG_ASYNC, 2341497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .cra_type = &crypto_ahash_type 2342497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper } 2343497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper }, 2344497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2345497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper DESC_HDR_SEL0_MDEUB | 2346497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper DESC_HDR_MODE0_MDEUB_SHA512, 2347497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper }, 234879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper { .type = CRYPTO_ALG_TYPE_AHASH, 234979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .alg.hash = { 235079b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .init = ahash_init, 235179b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .update = ahash_update, 235279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .final = ahash_final, 235379b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .finup = ahash_finup, 235479b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .digest = ahash_digest, 235579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .setkey = ahash_setkey, 235679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .halg.digestsize = MD5_DIGEST_SIZE, 235779b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .halg.base = { 235879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_name = "hmac(md5)", 235979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_driver_name = "hmac-md5-talitos", 236079b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_blocksize = MD5_BLOCK_SIZE, 236179b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_flags = CRYPTO_ALG_TYPE_AHASH | 236279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper CRYPTO_ALG_ASYNC, 236379b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_type = &crypto_ahash_type 236479b3a418e090248d00ceba40b81da9dfac753367Lee Nipper } 236579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper }, 236679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 236779b3a418e090248d00ceba40b81da9dfac753367Lee Nipper DESC_HDR_SEL0_MDEUA | 236879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper DESC_HDR_MODE0_MDEU_MD5, 236979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper }, 237079b3a418e090248d00ceba40b81da9dfac753367Lee Nipper { .type = CRYPTO_ALG_TYPE_AHASH, 237179b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .alg.hash = { 237279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .init = ahash_init, 237379b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .update = ahash_update, 237479b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .final = ahash_final, 237579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .finup = ahash_finup, 237679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .digest = ahash_digest, 237779b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .setkey = ahash_setkey, 237879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .halg.digestsize = SHA1_DIGEST_SIZE, 237979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .halg.base = { 238079b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_name = "hmac(sha1)", 238179b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_driver_name = "hmac-sha1-talitos", 238279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_blocksize = SHA1_BLOCK_SIZE, 238379b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_flags = CRYPTO_ALG_TYPE_AHASH | 238479b3a418e090248d00ceba40b81da9dfac753367Lee Nipper CRYPTO_ALG_ASYNC, 238579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_type = &crypto_ahash_type 238679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper } 238779b3a418e090248d00ceba40b81da9dfac753367Lee Nipper }, 238879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 238979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper DESC_HDR_SEL0_MDEUA | 239079b3a418e090248d00ceba40b81da9dfac753367Lee Nipper DESC_HDR_MODE0_MDEU_SHA1, 239179b3a418e090248d00ceba40b81da9dfac753367Lee Nipper }, 239279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper { .type = CRYPTO_ALG_TYPE_AHASH, 239379b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .alg.hash = { 239479b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .init = ahash_init, 239579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .update = ahash_update, 239679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .final = ahash_final, 239779b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .finup = ahash_finup, 239879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .digest = ahash_digest, 239979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .setkey = ahash_setkey, 240079b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .halg.digestsize = SHA224_DIGEST_SIZE, 240179b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .halg.base = { 240279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_name = "hmac(sha224)", 240379b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_driver_name = "hmac-sha224-talitos", 240479b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_blocksize = SHA224_BLOCK_SIZE, 240579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_flags = CRYPTO_ALG_TYPE_AHASH | 240679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper CRYPTO_ALG_ASYNC, 240779b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_type = &crypto_ahash_type 240879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper } 240979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper }, 241079b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 241179b3a418e090248d00ceba40b81da9dfac753367Lee Nipper DESC_HDR_SEL0_MDEUA | 241279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper DESC_HDR_MODE0_MDEU_SHA224, 241379b3a418e090248d00ceba40b81da9dfac753367Lee Nipper }, 241479b3a418e090248d00ceba40b81da9dfac753367Lee Nipper { .type = CRYPTO_ALG_TYPE_AHASH, 241579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .alg.hash = { 241679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .init = ahash_init, 241779b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .update = ahash_update, 241879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .final = ahash_final, 241979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .finup = ahash_finup, 242079b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .digest = ahash_digest, 242179b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .setkey = ahash_setkey, 242279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .halg.digestsize = SHA256_DIGEST_SIZE, 242379b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .halg.base = { 242479b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_name = "hmac(sha256)", 242579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_driver_name = "hmac-sha256-talitos", 242679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_blocksize = SHA256_BLOCK_SIZE, 242779b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_flags = CRYPTO_ALG_TYPE_AHASH | 242879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper CRYPTO_ALG_ASYNC, 242979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_type = &crypto_ahash_type 243079b3a418e090248d00ceba40b81da9dfac753367Lee Nipper } 243179b3a418e090248d00ceba40b81da9dfac753367Lee Nipper }, 243279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 243379b3a418e090248d00ceba40b81da9dfac753367Lee Nipper DESC_HDR_SEL0_MDEUA | 243479b3a418e090248d00ceba40b81da9dfac753367Lee Nipper DESC_HDR_MODE0_MDEU_SHA256, 243579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper }, 243679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper { .type = CRYPTO_ALG_TYPE_AHASH, 243779b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .alg.hash = { 243879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .init = ahash_init, 243979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .update = ahash_update, 244079b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .final = ahash_final, 244179b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .finup = ahash_finup, 244279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .digest = ahash_digest, 244379b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .setkey = ahash_setkey, 244479b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .halg.digestsize = SHA384_DIGEST_SIZE, 244579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .halg.base = { 244679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_name = "hmac(sha384)", 244779b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_driver_name = "hmac-sha384-talitos", 244879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_blocksize = SHA384_BLOCK_SIZE, 244979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_flags = CRYPTO_ALG_TYPE_AHASH | 245079b3a418e090248d00ceba40b81da9dfac753367Lee Nipper CRYPTO_ALG_ASYNC, 245179b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_type = &crypto_ahash_type 245279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper } 245379b3a418e090248d00ceba40b81da9dfac753367Lee Nipper }, 245479b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 245579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper DESC_HDR_SEL0_MDEUB | 245679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper DESC_HDR_MODE0_MDEUB_SHA384, 245779b3a418e090248d00ceba40b81da9dfac753367Lee Nipper }, 245879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper { .type = CRYPTO_ALG_TYPE_AHASH, 245979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .alg.hash = { 246079b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .init = ahash_init, 246179b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .update = ahash_update, 246279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .final = ahash_final, 246379b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .finup = ahash_finup, 246479b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .digest = ahash_digest, 246579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .setkey = ahash_setkey, 246679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .halg.digestsize = SHA512_DIGEST_SIZE, 246779b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .halg.base = { 246879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_name = "hmac(sha512)", 246979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_driver_name = "hmac-sha512-talitos", 247079b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_blocksize = SHA512_BLOCK_SIZE, 247179b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_flags = CRYPTO_ALG_TYPE_AHASH | 247279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper CRYPTO_ALG_ASYNC, 247379b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .cra_type = &crypto_ahash_type 247479b3a418e090248d00ceba40b81da9dfac753367Lee Nipper } 247579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper }, 247679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 247779b3a418e090248d00ceba40b81da9dfac753367Lee Nipper DESC_HDR_SEL0_MDEUB | 247879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper DESC_HDR_MODE0_MDEUB_SHA512, 247979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper } 24809c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips}; 24819c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 24829c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstruct talitos_crypto_alg { 24839c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct list_head entry; 24849c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct device *dev; 2485acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper struct talitos_alg_template algt; 24869c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips}; 24879c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 24889c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstatic int talitos_cra_init(struct crypto_tfm *tfm) 24899c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 24909c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct crypto_alg *alg = tfm->__crt_alg; 249119bbbc635523703ece28409e59694d5b512b819eKim Phillips struct talitos_crypto_alg *talitos_alg; 24929c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); 24935228f0f79e983c2b39c202c75af901ceb0003fc1Kim Phillips struct talitos_private *priv; 24949c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2495497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH) 2496497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper talitos_alg = container_of(__crypto_ahash_alg(alg), 2497497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct talitos_crypto_alg, 2498497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper algt.alg.hash); 2499497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper else 2500497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper talitos_alg = container_of(alg, struct talitos_crypto_alg, 2501497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper algt.alg.crypto); 250219bbbc635523703ece28409e59694d5b512b819eKim Phillips 25039c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* update context with ptr to dev */ 25049c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips ctx->dev = talitos_alg->dev; 250519bbbc635523703ece28409e59694d5b512b819eKim Phillips 25065228f0f79e983c2b39c202c75af901ceb0003fc1Kim Phillips /* assign SEC channel to tfm in round-robin fashion */ 25075228f0f79e983c2b39c202c75af901ceb0003fc1Kim Phillips priv = dev_get_drvdata(ctx->dev); 25085228f0f79e983c2b39c202c75af901ceb0003fc1Kim Phillips ctx->ch = atomic_inc_return(&priv->last_chan) & 25095228f0f79e983c2b39c202c75af901ceb0003fc1Kim Phillips (priv->num_channels - 1); 25105228f0f79e983c2b39c202c75af901ceb0003fc1Kim Phillips 25119c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* copy descriptor header template value */ 2512acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template; 25139c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2514602dba5a81790375f364e484cbc439f17157752eKim Phillips /* select done notification */ 2515602dba5a81790375f364e484cbc439f17157752eKim Phillips ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY; 2516602dba5a81790375f364e484cbc439f17157752eKim Phillips 2517497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper return 0; 2518497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper} 2519497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 2520497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipperstatic int talitos_cra_init_aead(struct crypto_tfm *tfm) 2521497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper{ 2522497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); 2523497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 2524497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper talitos_cra_init(tfm); 25259c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 25269c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* random first IV */ 252770bcaca75389a6c011ddc866eb1743b070a838b0Lee Nipper get_random_bytes(ctx->iv, TALITOS_MAX_IV_LENGTH); 25289c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 25299c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return 0; 25309c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 25319c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2532497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipperstatic int talitos_cra_init_ahash(struct crypto_tfm *tfm) 2533497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper{ 2534497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); 2535497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 2536497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper talitos_cra_init(tfm); 2537497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 2538497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper ctx->keylen = 0; 2539497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 2540497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper sizeof(struct talitos_ahash_req_ctx)); 2541497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 2542497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper return 0; 2543497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper} 2544497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper 25459c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips/* 25469c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * given the alg's descriptor header template, determine whether descriptor 25479c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * type and primary/secondary execution units required match the hw 25489c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips * capabilities description provided in the device tree node. 25499c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips */ 25509c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstatic int hw_supports(struct device *dev, __be32 desc_hdr_template) 25519c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 25529c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_private *priv = dev_get_drvdata(dev); 25539c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips int ret; 25549c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 25559c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) && 25569c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units); 25579c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 25589c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (SECONDARY_EU(desc_hdr_template)) 25599c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips ret = ret && (1 << SECONDARY_EU(desc_hdr_template) 25609c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips & priv->exec_units); 25619c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 25629c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return ret; 25639c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 25649c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 25652dc11581376829303b98eadb2de253bee065a56aGrant Likelystatic int talitos_remove(struct platform_device *ofdev) 25669c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 25679c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct device *dev = &ofdev->dev; 25689c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_private *priv = dev_get_drvdata(dev); 25699c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_crypto_alg *t_alg, *n; 25709c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips int i; 25719c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 25729c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) { 2573acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper switch (t_alg->algt.type) { 2574acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper case CRYPTO_ALG_TYPE_ABLKCIPHER: 2575acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper case CRYPTO_ALG_TYPE_AEAD: 2576acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper crypto_unregister_alg(&t_alg->algt.alg.crypto); 2577acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper break; 2578acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper case CRYPTO_ALG_TYPE_AHASH: 2579acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper crypto_unregister_ahash(&t_alg->algt.alg.hash); 2580acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper break; 2581acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper } 25829c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips list_del(&t_alg->entry); 25839c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips kfree(t_alg); 25849c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 25859c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 25869c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (hw_supports(dev, DESC_HDR_SEL0_RNG)) 25879c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips talitos_unregister_rng(dev); 25889c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 25894b992628812137e30cda3510510cf3c052345b30Kim Phillips for (i = 0; i < priv->num_channels; i++) 25900b798247453299c895e3fa1629101dd5e94901b2Kim Phillips kfree(priv->chan[i].fifo); 25919c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 25924b992628812137e30cda3510510cf3c052345b30Kim Phillips kfree(priv->chan); 25939c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2594c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips for (i = 0; i < 2; i++) 25952cdba3cf6ffc1fbf880a6fbfa9e7bb757e3d6526Kim Phillips if (priv->irq[i]) { 2596c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips free_irq(priv->irq[i], dev); 2597c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips irq_dispose_mapping(priv->irq[i]); 2598c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips } 25999c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2600c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips tasklet_kill(&priv->done_task[0]); 26012cdba3cf6ffc1fbf880a6fbfa9e7bb757e3d6526Kim Phillips if (priv->irq[1]) 2602c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips tasklet_kill(&priv->done_task[1]); 26039c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 26049c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips iounmap(priv->reg); 26059c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 26069c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_set_drvdata(dev, NULL); 26079c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 26089c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips kfree(priv); 26099c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 26109c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return 0; 26119c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 26129c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 26139c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipsstatic struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, 26149c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_alg_template 26159c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips *template) 26169c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 261760f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips struct talitos_private *priv = dev_get_drvdata(dev); 26189c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_crypto_alg *t_alg; 26199c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct crypto_alg *alg; 26209c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 26219c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL); 26229c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (!t_alg) 26239c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return ERR_PTR(-ENOMEM); 26249c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2625acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper t_alg->algt = *template; 2626acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper 2627acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper switch (t_alg->algt.type) { 2628acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper case CRYPTO_ALG_TYPE_ABLKCIPHER: 2629497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper alg = &t_alg->algt.alg.crypto; 2630497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper alg->cra_init = talitos_cra_init; 2631497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper break; 2632acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper case CRYPTO_ALG_TYPE_AEAD: 2633acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper alg = &t_alg->algt.alg.crypto; 2634497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper alg->cra_init = talitos_cra_init_aead; 2635acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper break; 2636acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper case CRYPTO_ALG_TYPE_AHASH: 2637acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper alg = &t_alg->algt.alg.hash.halg.base; 2638497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper alg->cra_init = talitos_cra_init_ahash; 263979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper if (!(priv->features & TALITOS_FTR_HMAC_OK) && 26400b2730d8d8b38e009607d5a094d48fcce73af547Kim Phillips !strncmp(alg->cra_name, "hmac", 4)) { 26410b2730d8d8b38e009607d5a094d48fcce73af547Kim Phillips kfree(t_alg); 264279b3a418e090248d00ceba40b81da9dfac753367Lee Nipper return ERR_PTR(-ENOTSUPP); 26430b2730d8d8b38e009607d5a094d48fcce73af547Kim Phillips } 264460f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) && 264579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper (!strcmp(alg->cra_name, "sha224") || 264679b3a418e090248d00ceba40b81da9dfac753367Lee Nipper !strcmp(alg->cra_name, "hmac(sha224)"))) { 264760f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips t_alg->algt.alg.hash.init = ahash_init_sha224_swinit; 264860f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips t_alg->algt.desc_hdr_template = 264960f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 265060f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips DESC_HDR_SEL0_MDEUA | 265160f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips DESC_HDR_MODE0_MDEU_SHA256; 265260f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips } 2653497f2e6b8b21407625a4fb34bc04b50eff098085Lee Nipper break; 26541d11911a8c572041880c8d86229f6ed971f6f7e2Kim Phillips default: 26551d11911a8c572041880c8d86229f6ed971f6f7e2Kim Phillips dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type); 26561d11911a8c572041880c8d86229f6ed971f6f7e2Kim Phillips return ERR_PTR(-EINVAL); 2657acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper } 26589c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 26599c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips alg->cra_module = THIS_MODULE; 26609c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips alg->cra_priority = TALITOS_CRA_PRIORITY; 26619c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips alg->cra_alignmask = 0; 26629c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips alg->cra_ctxsize = sizeof(struct talitos_ctx); 2663d912bb7677f46d78a3cde8a4afd45a3fca4b34e9Nikos Mavrogiannopoulos alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY; 26649c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 26659c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips t_alg->dev = dev; 26669c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 26679c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return t_alg; 26689c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 26699c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2670c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillipsstatic int talitos_probe_irq(struct platform_device *ofdev) 2671c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips{ 2672c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips struct device *dev = &ofdev->dev; 2673c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips struct device_node *np = ofdev->dev.of_node; 2674c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips struct talitos_private *priv = dev_get_drvdata(dev); 2675c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips int err; 2676c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips 2677c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips priv->irq[0] = irq_of_parse_and_map(np, 0); 26782cdba3cf6ffc1fbf880a6fbfa9e7bb757e3d6526Kim Phillips if (!priv->irq[0]) { 2679c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips dev_err(dev, "failed to map irq\n"); 2680c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips return -EINVAL; 2681c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips } 2682c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips 2683c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips priv->irq[1] = irq_of_parse_and_map(np, 1); 2684c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips 2685c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips /* get the primary irq line */ 26862cdba3cf6ffc1fbf880a6fbfa9e7bb757e3d6526Kim Phillips if (!priv->irq[1]) { 2687c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips err = request_irq(priv->irq[0], talitos_interrupt_4ch, 0, 2688c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips dev_driver_string(dev), dev); 2689c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips goto primary_out; 2690c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips } 2691c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips 2692c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips err = request_irq(priv->irq[0], talitos_interrupt_ch0_2, 0, 2693c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips dev_driver_string(dev), dev); 2694c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips if (err) 2695c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips goto primary_out; 2696c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips 2697c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips /* get the secondary irq line */ 2698c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips err = request_irq(priv->irq[1], talitos_interrupt_ch1_3, 0, 2699c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips dev_driver_string(dev), dev); 2700c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips if (err) { 2701c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips dev_err(dev, "failed to request secondary irq\n"); 2702c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips irq_dispose_mapping(priv->irq[1]); 27032cdba3cf6ffc1fbf880a6fbfa9e7bb757e3d6526Kim Phillips priv->irq[1] = 0; 2704c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips } 2705c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips 2706c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips return err; 2707c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips 2708c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillipsprimary_out: 2709c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips if (err) { 2710c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips dev_err(dev, "failed to request primary irq\n"); 2711c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips irq_dispose_mapping(priv->irq[0]); 27122cdba3cf6ffc1fbf880a6fbfa9e7bb757e3d6526Kim Phillips priv->irq[0] = 0; 2713c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips } 2714c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips 2715c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips return err; 2716c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips} 2717c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips 27181c48a5c93da63132b92c4bbcd18e690c51539df6Grant Likelystatic int talitos_probe(struct platform_device *ofdev) 27199c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips{ 27209c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct device *dev = &ofdev->dev; 272161c7a080a5a061c976988fd4b844dfb468dda255Grant Likely struct device_node *np = ofdev->dev.of_node; 27229c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_private *priv; 27239c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips const unsigned int *prop; 27249c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips int i, err; 27259c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 27269c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL); 27279c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (!priv) 27289c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return -ENOMEM; 27299c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 27309c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_set_drvdata(dev, priv); 27319c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 27329c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips priv->ofdev = ofdev; 27339c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2734511d63cb19329235bc9298b64010ec494b5e1408Horia Geanta spin_lock_init(&priv->reg_lock); 2735511d63cb19329235bc9298b64010ec494b5e1408Horia Geanta 2736c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips err = talitos_probe_irq(ofdev); 2737c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips if (err) 27389c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips goto err_out; 27399c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 27402cdba3cf6ffc1fbf880a6fbfa9e7bb757e3d6526Kim Phillips if (!priv->irq[1]) { 2741c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips tasklet_init(&priv->done_task[0], talitos_done_4ch, 2742c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips (unsigned long)dev); 2743c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips } else { 2744c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips tasklet_init(&priv->done_task[0], talitos_done_ch0_2, 2745c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips (unsigned long)dev); 2746c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips tasklet_init(&priv->done_task[1], talitos_done_ch1_3, 2747c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips (unsigned long)dev); 27489c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 27499c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2750c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips INIT_LIST_HEAD(&priv->alg_list); 2751c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips 27529c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips priv->reg = of_iomap(np, 0); 27539c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (!priv->reg) { 27549c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "failed to of_iomap\n"); 27559c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips err = -ENOMEM; 27569c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips goto err_out; 27579c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 27589c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 27599c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* get SEC version capabilities from device tree */ 27609c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips prop = of_get_property(np, "fsl,num-channels", NULL); 27619c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (prop) 27629c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips priv->num_channels = *prop; 27639c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 27649c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips prop = of_get_property(np, "fsl,channel-fifo-len", NULL); 27659c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (prop) 27669c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips priv->chfifo_len = *prop; 27679c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 27689c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips prop = of_get_property(np, "fsl,exec-units-mask", NULL); 27699c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (prop) 27709c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips priv->exec_units = *prop; 27719c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 27729c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips prop = of_get_property(np, "fsl,descriptor-types-mask", NULL); 27739c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (prop) 27749c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips priv->desc_types = *prop; 27759c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 27769c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len || 27779c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips !priv->exec_units || !priv->desc_types) { 27789c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "invalid property data in device tree node\n"); 27799c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips err = -EINVAL; 27809c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips goto err_out; 27819c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 27829c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2783f3c85bc1bc72b4cc8d58664a490a9d42bdb6565aLee Nipper if (of_device_is_compatible(np, "fsl,sec3.0")) 2784f3c85bc1bc72b4cc8d58664a490a9d42bdb6565aLee Nipper priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT; 2785f3c85bc1bc72b4cc8d58664a490a9d42bdb6565aLee Nipper 2786fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips if (of_device_is_compatible(np, "fsl,sec2.1")) 278760f208d7836216885cdcd6f77a02f31dbc66f169Kim Phillips priv->features |= TALITOS_FTR_HW_AUTH_CHECK | 278879b3a418e090248d00ceba40b81da9dfac753367Lee Nipper TALITOS_FTR_SHA224_HWINIT | 278979b3a418e090248d00ceba40b81da9dfac753367Lee Nipper TALITOS_FTR_HMAC_OK; 2790fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70Kim Phillips 27914b992628812137e30cda3510510cf3c052345b30Kim Phillips priv->chan = kzalloc(sizeof(struct talitos_channel) * 27924b992628812137e30cda3510510cf3c052345b30Kim Phillips priv->num_channels, GFP_KERNEL); 27934b992628812137e30cda3510510cf3c052345b30Kim Phillips if (!priv->chan) { 27944b992628812137e30cda3510510cf3c052345b30Kim Phillips dev_err(dev, "failed to allocate channel management space\n"); 27959c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips err = -ENOMEM; 27969c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips goto err_out; 27979c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 27989c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2799c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips for (i = 0; i < priv->num_channels; i++) { 2800c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips priv->chan[i].reg = priv->reg + TALITOS_CH_STRIDE * (i + 1); 28012cdba3cf6ffc1fbf880a6fbfa9e7bb757e3d6526Kim Phillips if (!priv->irq[1] || !(i & 1)) 2802c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips priv->chan[i].reg += TALITOS_CH_BASE_OFFSET; 2803c3e337f88a5b3784cb3c806ffd650d06adff1ea5Kim Phillips } 2804ad42d5fc85383278663ecb58a24f6547ad0ba735Kim Phillips 28059c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips for (i = 0; i < priv->num_channels; i++) { 28064b992628812137e30cda3510510cf3c052345b30Kim Phillips spin_lock_init(&priv->chan[i].head_lock); 28074b992628812137e30cda3510510cf3c052345b30Kim Phillips spin_lock_init(&priv->chan[i].tail_lock); 28089c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 28099c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 28109c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips priv->fifo_len = roundup_pow_of_two(priv->chfifo_len); 28119c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 28129c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips for (i = 0; i < priv->num_channels; i++) { 28134b992628812137e30cda3510510cf3c052345b30Kim Phillips priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) * 28144b992628812137e30cda3510510cf3c052345b30Kim Phillips priv->fifo_len, GFP_KERNEL); 28154b992628812137e30cda3510510cf3c052345b30Kim Phillips if (!priv->chan[i].fifo) { 28169c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "failed to allocate request fifo %d\n", i); 28179c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips err = -ENOMEM; 28189c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips goto err_out; 28199c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 28209c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 28219c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2822ec6644d6325b5a38525f1d5b20fd4bf7db05cf2aKim Phillips for (i = 0; i < priv->num_channels; i++) 28234b992628812137e30cda3510510cf3c052345b30Kim Phillips atomic_set(&priv->chan[i].submit_count, 28244b992628812137e30cda3510510cf3c052345b30Kim Phillips -(priv->chfifo_len - 1)); 28259c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 282681eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips dma_set_mask(dev, DMA_BIT_MASK(36)); 282781eb024c7e63f53b871797f6e2defccfd008dcd4Kim Phillips 28289c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* reset and initialize the h/w */ 28299c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips err = init_device(dev); 28309c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (err) { 28319c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "failed to initialize device\n"); 28329c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips goto err_out; 28339c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 28349c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 28359c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* register the RNG, if available */ 28369c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (hw_supports(dev, DESC_HDR_SEL0_RNG)) { 28379c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips err = talitos_register_rng(dev); 28389c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (err) { 28399c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "failed to register hwrng: %d\n", err); 28409c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips goto err_out; 28419c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } else 28429c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_info(dev, "hwrng\n"); 28439c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 28449c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 28459c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips /* register crypto algorithms the device supports */ 28469c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 28479c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (hw_supports(dev, driver_algs[i].desc_hdr_template)) { 28489c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips struct talitos_crypto_alg *t_alg; 2849acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper char *name = NULL; 28509c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 28519c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips t_alg = talitos_alg_alloc(dev, &driver_algs[i]); 28529c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (IS_ERR(t_alg)) { 28539c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips err = PTR_ERR(t_alg); 28540b2730d8d8b38e009607d5a094d48fcce73af547Kim Phillips if (err == -ENOTSUPP) 285579b3a418e090248d00ceba40b81da9dfac753367Lee Nipper continue; 28569c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips goto err_out; 28579c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 28589c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2859acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper switch (t_alg->algt.type) { 2860acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper case CRYPTO_ALG_TYPE_ABLKCIPHER: 2861acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper case CRYPTO_ALG_TYPE_AEAD: 2862acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper err = crypto_register_alg( 2863acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper &t_alg->algt.alg.crypto); 2864acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper name = t_alg->algt.alg.crypto.cra_driver_name; 2865acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper break; 2866acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper case CRYPTO_ALG_TYPE_AHASH: 2867acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper err = crypto_register_ahash( 2868acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper &t_alg->algt.alg.hash); 2869acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper name = 2870acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper t_alg->algt.alg.hash.halg.base.cra_driver_name; 2871acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper break; 2872acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper } 28739c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips if (err) { 28749c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips dev_err(dev, "%s alg registration failed\n", 2875acbf7c627fb59dfea975f7aafeaba97921085061Lee Nipper name); 28769c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips kfree(t_alg); 28775b859b6ebb18b37244d44b5300bf765694b7303cKim Phillips } else 28789c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips list_add_tail(&t_alg->entry, &priv->alg_list); 28799c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 28809c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips } 28815b859b6ebb18b37244d44b5300bf765694b7303cKim Phillips if (!list_empty(&priv->alg_list)) 28825b859b6ebb18b37244d44b5300bf765694b7303cKim Phillips dev_info(dev, "%s algorithms registered in /proc/crypto\n", 28835b859b6ebb18b37244d44b5300bf765694b7303cKim Phillips (char *)of_get_property(np, "compatible", NULL)); 28849c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 28859c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return 0; 28869c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 28879c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillipserr_out: 28889c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips talitos_remove(ofdev); 28899c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 28909c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips return err; 28919c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips} 28929c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 28936c3f975a4cafaf4dcc9795385da6d42caa37ddebMárton Némethstatic const struct of_device_id talitos_match[] = { 28949c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips { 28959c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips .compatible = "fsl,sec2.0", 28969c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips }, 28979c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips {}, 28989c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips}; 28999c4a79653b35efc9d6790c295e22f79f4b361125Kim PhillipsMODULE_DEVICE_TABLE(of, talitos_match); 29009c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 29011c48a5c93da63132b92c4bbcd18e690c51539df6Grant Likelystatic struct platform_driver talitos_driver = { 29024018294b53d1dae026880e45f174c1cc63b5d435Grant Likely .driver = { 29034018294b53d1dae026880e45f174c1cc63b5d435Grant Likely .name = "talitos", 29044018294b53d1dae026880e45f174c1cc63b5d435Grant Likely .owner = THIS_MODULE, 29054018294b53d1dae026880e45f174c1cc63b5d435Grant Likely .of_match_table = talitos_match, 29064018294b53d1dae026880e45f174c1cc63b5d435Grant Likely }, 29079c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips .probe = talitos_probe, 2908596f1034190565529e507e1eb8df6def1c9f5560Al Viro .remove = talitos_remove, 29099c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips}; 29109c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 2911741e8c2d8177eca656bc015ef83ab84d817edf8cAxel Linmodule_platform_driver(talitos_driver); 29129c4a79653b35efc9d6790c295e22f79f4b361125Kim Phillips 29139c4a79653b35efc9d6790c295e22f79f4b361125Kim PhillipsMODULE_LICENSE("GPL"); 29149c4a79653b35efc9d6790c295e22f79f4b361125Kim PhillipsMODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>"); 29159c4a79653b35efc9d6790c295e22f79f4b361125Kim PhillipsMODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver"); 2916