1ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen/* 2ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * Multi buffer SHA1 algorithm Glue Code 3ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * 4ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * This file is provided under a dual BSD/GPLv2 license. When using or 5ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * redistributing this file, you may do so under either license. 6ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * 7ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * GPL LICENSE SUMMARY 8ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * 9ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * Copyright(c) 2014 Intel Corporation. 10ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * 11ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * This program is free software; you can redistribute it and/or modify 12ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * it under the terms of version 2 of the GNU General Public License as 13ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * published by the Free Software Foundation. 14ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * 15ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * This program is distributed in the hope that it will be useful, but 16ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * WITHOUT ANY WARRANTY; without even the implied warranty of 17ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 18ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * General Public License for more details. 19ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * 20ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * Contact Information: 21ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * Tim Chen <tim.c.chen@linux.intel.com> 22ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * 23ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * BSD LICENSE 24ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * 25ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * Copyright(c) 2014 Intel Corporation. 26ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * 27ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * Redistribution and use in source and binary forms, with or without 28ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * modification, are permitted provided that the following conditions 29ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * are met: 30ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * 31ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * * Redistributions of source code must retain the above copyright 32ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * notice, this list of conditions and the following disclaimer. 33ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * * Redistributions in binary form must reproduce the above copyright 34ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * notice, this list of conditions and the following disclaimer in 35ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * the documentation and/or other materials provided with the 36ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * distribution. 37ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * * Neither the name of Intel Corporation nor the names of its 38ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * contributors may be used to endorse or promote products derived 39ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * from this software without specific prior written permission. 40ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * 41ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 42ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 43ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 44ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 45ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 46ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 47ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 48ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 49ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 50ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 51ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 52ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen */ 53ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 54ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 55ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 56ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen#include <crypto/internal/hash.h> 57ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen#include <linux/init.h> 58ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen#include <linux/module.h> 59ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen#include <linux/mm.h> 60ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen#include <linux/cryptohash.h> 61ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen#include <linux/types.h> 62ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen#include <linux/list.h> 63ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen#include <crypto/scatterwalk.h> 64ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen#include <crypto/sha.h> 65ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen#include <crypto/mcryptd.h> 66ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen#include <crypto/crypto_wq.h> 67ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen#include <asm/byteorder.h> 68ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen#include <asm/i387.h> 69ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen#include <asm/xcr.h> 70ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen#include <asm/xsave.h> 71ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen#include <linux/hardirq.h> 72ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen#include <asm/fpu-internal.h> 73ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen#include "sha_mb_ctx.h" 74ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 75ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen#define FLUSH_INTERVAL 1000 /* in usec */ 76ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 774c1948fc471d444b15dfaca73ea0c9b77c2d5505Fengguang Wustatic struct mcryptd_alg_state sha1_mb_alg_state; 78ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 79ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstruct sha1_mb_ctx { 80ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_ahash *mcryptd_tfm; 81ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen}; 82ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 83ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic inline struct mcryptd_hash_request_ctx *cast_hash_to_mcryptd_ctx(struct sha1_hash_ctx *hash_ctx) 84ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 85ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct shash_desc *desc; 86ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 87ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen desc = container_of((void *) hash_ctx, struct shash_desc, __ctx); 88ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return container_of(desc, struct mcryptd_hash_request_ctx, desc); 89ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 90ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 91ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic inline struct ahash_request *cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx *ctx) 92ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 93ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return container_of((void *) ctx, struct ahash_request, __ctx); 94ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 95ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 96ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic void req_ctx_init(struct mcryptd_hash_request_ctx *rctx, 97ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct shash_desc *desc) 98ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 99ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx->flag = HASH_UPDATE; 100ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 101ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 1024c1948fc471d444b15dfaca73ea0c9b77c2d5505Fengguang Wustatic asmlinkage void (*sha1_job_mgr_init)(struct sha1_mb_mgr *state); 1034c1948fc471d444b15dfaca73ea0c9b77c2d5505Fengguang Wustatic asmlinkage struct job_sha1* (*sha1_job_mgr_submit)(struct sha1_mb_mgr *state, 104ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct job_sha1 *job); 1054c1948fc471d444b15dfaca73ea0c9b77c2d5505Fengguang Wustatic asmlinkage struct job_sha1* (*sha1_job_mgr_flush)(struct sha1_mb_mgr *state); 1064c1948fc471d444b15dfaca73ea0c9b77c2d5505Fengguang Wustatic asmlinkage struct job_sha1* (*sha1_job_mgr_get_comp_job)(struct sha1_mb_mgr *state); 107ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 108ad61e042e9151b55b393d5875e467e7fe0c7470cTim Cheninline void sha1_init_digest(uint32_t *digest) 109ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 110ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen static const uint32_t initial_digest[SHA1_DIGEST_LENGTH] = {SHA1_H0, 111ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }; 112ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen memcpy(digest, initial_digest, sizeof(initial_digest)); 113ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 114ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 115ad61e042e9151b55b393d5875e467e7fe0c7470cTim Cheninline uint32_t sha1_pad(uint8_t padblock[SHA1_BLOCK_SIZE * 2], 116ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen uint32_t total_len) 117ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 118ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen uint32_t i = total_len & (SHA1_BLOCK_SIZE - 1); 119ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 120ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen memset(&padblock[i], 0, SHA1_BLOCK_SIZE); 121ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen padblock[i] = 0x80; 122ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 123ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen i += ((SHA1_BLOCK_SIZE - 1) & 124ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen (0 - (total_len + SHA1_PADLENGTHFIELD_SIZE + 1))) 125ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen + 1 + SHA1_PADLENGTHFIELD_SIZE; 126ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 127ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen#if SHA1_PADLENGTHFIELD_SIZE == 16 128ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen *((uint64_t *) &padblock[i - 16]) = 0; 129ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen#endif 130ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 131ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen *((uint64_t *) &padblock[i - 8]) = cpu_to_be64(total_len << 3); 132ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 133ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* Number of extra blocks to hash */ 134ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return i >> SHA1_LOG2_BLOCK_SIZE; 135ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 136ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 137ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic struct sha1_hash_ctx *sha1_ctx_mgr_resubmit(struct sha1_ctx_mgr *mgr, struct sha1_hash_ctx *ctx) 138ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 139ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen while (ctx) { 140ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (ctx->status & HASH_CTX_STS_COMPLETE) { 141ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* Clear PROCESSING bit */ 142ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->status = HASH_CTX_STS_COMPLETE; 143ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return ctx; 144ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 145ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 146ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* 147ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * If the extra blocks are empty, begin hashing what remains 148ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * in the user's buffer. 149ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen */ 150ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (ctx->partial_block_buffer_length == 0 && 151ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->incoming_buffer_length) { 152ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 153ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen const void *buffer = ctx->incoming_buffer; 154ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen uint32_t len = ctx->incoming_buffer_length; 155ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen uint32_t copy_len; 156ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 157ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* 158ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * Only entire blocks can be hashed. 159ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * Copy remainder to extra blocks buffer. 160ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen */ 161ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen copy_len = len & (SHA1_BLOCK_SIZE-1); 162ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 163ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (copy_len) { 164ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen len -= copy_len; 165ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen memcpy(ctx->partial_block_buffer, 166ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ((const char *) buffer + len), 167ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen copy_len); 168ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->partial_block_buffer_length = copy_len; 169ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 170ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 171ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->incoming_buffer_length = 0; 172ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 173ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* len should be a multiple of the block size now */ 174ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen assert((len % SHA1_BLOCK_SIZE) == 0); 175ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 176ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* Set len to the number of blocks to be hashed */ 177ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen len >>= SHA1_LOG2_BLOCK_SIZE; 178ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 179ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (len) { 180ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 181ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->job.buffer = (uint8_t *) buffer; 182ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->job.len = len; 183ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx = (struct sha1_hash_ctx *) sha1_job_mgr_submit(&mgr->mgr, 184ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen &ctx->job); 185ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen continue; 186ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 187ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 188ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 189ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* 190ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * If the extra blocks are not empty, then we are 191ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * either on the last block(s) or we need more 192ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * user input before continuing. 193ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen */ 194ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (ctx->status & HASH_CTX_STS_LAST) { 195ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 196ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen uint8_t *buf = ctx->partial_block_buffer; 197ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen uint32_t n_extra_blocks = sha1_pad(buf, ctx->total_length); 198ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 199ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->status = (HASH_CTX_STS_PROCESSING | 200ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen HASH_CTX_STS_COMPLETE); 201ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->job.buffer = buf; 202ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->job.len = (uint32_t) n_extra_blocks; 203ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx = (struct sha1_hash_ctx *) sha1_job_mgr_submit(&mgr->mgr, &ctx->job); 204ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen continue; 205ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 206ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 207ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (ctx) 208ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->status = HASH_CTX_STS_IDLE; 209ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return ctx; 210ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 211ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 212ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return NULL; 213ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 214ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 2154c1948fc471d444b15dfaca73ea0c9b77c2d5505Fengguang Wustatic struct sha1_hash_ctx *sha1_ctx_mgr_get_comp_ctx(struct sha1_ctx_mgr *mgr) 216ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 217ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* 218ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * If get_comp_job returns NULL, there are no jobs complete. 219ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * If get_comp_job returns a job, verify that it is safe to return to the user. 220ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * If it is not ready, resubmit the job to finish processing. 221ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * If sha1_ctx_mgr_resubmit returned a job, it is ready to be returned. 222ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * Otherwise, all jobs currently being managed by the hash_ctx_mgr still need processing. 223ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen */ 224ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct sha1_hash_ctx *ctx; 225ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 226ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx = (struct sha1_hash_ctx *) sha1_job_mgr_get_comp_job(&mgr->mgr); 227ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return sha1_ctx_mgr_resubmit(mgr, ctx); 228ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 229ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 2304c1948fc471d444b15dfaca73ea0c9b77c2d5505Fengguang Wustatic void sha1_ctx_mgr_init(struct sha1_ctx_mgr *mgr) 231ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 232ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha1_job_mgr_init(&mgr->mgr); 233ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 234ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 2354c1948fc471d444b15dfaca73ea0c9b77c2d5505Fengguang Wustatic struct sha1_hash_ctx *sha1_ctx_mgr_submit(struct sha1_ctx_mgr *mgr, 236ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct sha1_hash_ctx *ctx, 237ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen const void *buffer, 238ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen uint32_t len, 239ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen int flags) 240ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 241ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (flags & (~HASH_ENTIRE)) { 242ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* User should not pass anything other than FIRST, UPDATE, or LAST */ 243ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->error = HASH_CTX_ERROR_INVALID_FLAGS; 244ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return ctx; 245ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 246ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 247ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (ctx->status & HASH_CTX_STS_PROCESSING) { 248ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* Cannot submit to a currently processing job. */ 249ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->error = HASH_CTX_ERROR_ALREADY_PROCESSING; 250ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return ctx; 251ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 252ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 253ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if ((ctx->status & HASH_CTX_STS_COMPLETE) && !(flags & HASH_FIRST)) { 254ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* Cannot update a finished job. */ 255ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->error = HASH_CTX_ERROR_ALREADY_COMPLETED; 256ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return ctx; 257ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 258ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 259ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 260ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (flags & HASH_FIRST) { 261ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* Init digest */ 262ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha1_init_digest(ctx->job.result_digest); 263ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 264ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* Reset byte counter */ 265ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->total_length = 0; 266ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 267ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* Clear extra blocks */ 268ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->partial_block_buffer_length = 0; 269ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 270ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 271ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* If we made it here, there were no errors during this call to submit */ 272ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->error = HASH_CTX_ERROR_NONE; 273ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 274ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* Store buffer ptr info from user */ 275ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->incoming_buffer = buffer; 276ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->incoming_buffer_length = len; 277ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 278ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* Store the user's request flags and mark this ctx as currently being processed. */ 279ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->status = (flags & HASH_LAST) ? 280ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen (HASH_CTX_STS_PROCESSING | HASH_CTX_STS_LAST) : 281ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen HASH_CTX_STS_PROCESSING; 282ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 283ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* Advance byte counter */ 284ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->total_length += len; 285ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 286ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* 287ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * If there is anything currently buffered in the extra blocks, 288ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * append to it until it contains a whole block. 289ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * Or if the user's buffer contains less than a whole block, 290ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * append as much as possible to the extra block. 291ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen */ 292ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if ((ctx->partial_block_buffer_length) | (len < SHA1_BLOCK_SIZE)) { 293ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* Compute how many bytes to copy from user buffer into extra block */ 294ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen uint32_t copy_len = SHA1_BLOCK_SIZE - ctx->partial_block_buffer_length; 295ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (len < copy_len) 296ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen copy_len = len; 297ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 298ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (copy_len) { 299ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* Copy and update relevant pointers and counters */ 300ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen memcpy(&ctx->partial_block_buffer[ctx->partial_block_buffer_length], 301ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen buffer, copy_len); 302ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 303ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->partial_block_buffer_length += copy_len; 304ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->incoming_buffer = (const void *)((const char *)buffer + copy_len); 305ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->incoming_buffer_length = len - copy_len; 306ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 307ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 308ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* The extra block should never contain more than 1 block here */ 309ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen assert(ctx->partial_block_buffer_length <= SHA1_BLOCK_SIZE); 310ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 311ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* If the extra block buffer contains exactly 1 block, it can be hashed. */ 312ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (ctx->partial_block_buffer_length >= SHA1_BLOCK_SIZE) { 313ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->partial_block_buffer_length = 0; 314ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 315ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->job.buffer = ctx->partial_block_buffer; 316ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->job.len = 1; 317ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx = (struct sha1_hash_ctx *) sha1_job_mgr_submit(&mgr->mgr, &ctx->job); 318ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 319ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 320ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 321ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return sha1_ctx_mgr_resubmit(mgr, ctx); 322ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 323ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 3244c1948fc471d444b15dfaca73ea0c9b77c2d5505Fengguang Wustatic struct sha1_hash_ctx *sha1_ctx_mgr_flush(struct sha1_ctx_mgr *mgr) 325ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 326ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct sha1_hash_ctx *ctx; 327ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 328ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen while (1) { 329ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx = (struct sha1_hash_ctx *) sha1_job_mgr_flush(&mgr->mgr); 330ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 331ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* If flush returned 0, there are no more jobs in flight. */ 332ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (!ctx) 333ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return NULL; 334ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 335ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* 336ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * If flush returned a job, resubmit the job to finish processing. 337ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen */ 338ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx = sha1_ctx_mgr_resubmit(mgr, ctx); 339ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 340ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* 341ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * If sha1_ctx_mgr_resubmit returned a job, it is ready to be returned. 342ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * Otherwise, all jobs currently being managed by the sha1_ctx_mgr 343ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * still need processing. Loop. 344ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen */ 345ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (ctx) 346ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return ctx; 347ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 348ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 349ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 350ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic int sha1_mb_init(struct shash_desc *desc) 351ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 352ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct sha1_hash_ctx *sctx = shash_desc_ctx(desc); 353ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 354ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen hash_ctx_init(sctx); 355ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sctx->job.result_digest[0] = SHA1_H0; 356ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sctx->job.result_digest[1] = SHA1_H1; 357ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sctx->job.result_digest[2] = SHA1_H2; 358ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sctx->job.result_digest[3] = SHA1_H3; 359ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sctx->job.result_digest[4] = SHA1_H4; 360ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sctx->total_length = 0; 361ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sctx->partial_block_buffer_length = 0; 362ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sctx->status = HASH_CTX_STS_IDLE; 363ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 364ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return 0; 365ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 366ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 367ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic int sha1_mb_set_results(struct mcryptd_hash_request_ctx *rctx) 368ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 369ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen int i; 370ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct sha1_hash_ctx *sctx = shash_desc_ctx(&rctx->desc); 371ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen __be32 *dst = (__be32 *) rctx->out; 372ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 373ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen for (i = 0; i < 5; ++i) 374ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen dst[i] = cpu_to_be32(sctx->job.result_digest[i]); 375ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 376ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return 0; 377ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 378ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 379ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx, 380ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_alg_cstate *cstate, bool flush) 381ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 382ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen int flag = HASH_UPDATE; 383ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen int nbytes, err = 0; 384ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_hash_request_ctx *rctx = *ret_rctx; 385ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct sha1_hash_ctx *sha_ctx; 386ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 387ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* more work ? */ 388ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen while (!(rctx->flag & HASH_DONE)) { 389ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen nbytes = crypto_ahash_walk_done(&rctx->walk, 0); 390ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (nbytes < 0) { 391ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen err = nbytes; 392ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen goto out; 393ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 394ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* check if the walk is done */ 395ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (crypto_ahash_walk_last(&rctx->walk)) { 396ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx->flag |= HASH_DONE; 397ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (rctx->flag & HASH_FINAL) 398ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen flag |= HASH_LAST; 399ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 400ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 401ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha_ctx = (struct sha1_hash_ctx *) shash_desc_ctx(&rctx->desc); 402ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen kernel_fpu_begin(); 403ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, nbytes, flag); 404ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (!sha_ctx) { 405ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (flush) 406ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha_ctx = sha1_ctx_mgr_flush(cstate->mgr); 407ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 408ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen kernel_fpu_end(); 409ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (sha_ctx) 410ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx = cast_hash_to_mcryptd_ctx(sha_ctx); 411ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen else { 412ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx = NULL; 413ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen goto out; 414ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 415ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 416ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 417ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* copy the results */ 418ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (rctx->flag & HASH_FINAL) 419ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha1_mb_set_results(rctx); 420ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 421ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenout: 422ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen *ret_rctx = rctx; 423ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return err; 424ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 425ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 426ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic int sha_complete_job(struct mcryptd_hash_request_ctx *rctx, 427ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_alg_cstate *cstate, 428ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen int err) 429ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 430ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); 431ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct sha1_hash_ctx *sha_ctx; 432ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_hash_request_ctx *req_ctx; 433ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen int ret; 434ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 435ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* remove from work list */ 436ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen spin_lock(&cstate->work_lock); 437ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen list_del(&rctx->waiter); 438ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen spin_unlock(&cstate->work_lock); 439ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 440ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (irqs_disabled()) 441ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx->complete(&req->base, err); 442ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen else { 443ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen local_bh_disable(); 444ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx->complete(&req->base, err); 445ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen local_bh_enable(); 446ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 447ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 448ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* check to see if there are other jobs that are done */ 449ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha_ctx = sha1_ctx_mgr_get_comp_ctx(cstate->mgr); 450ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen while (sha_ctx) { 451ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx); 452ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ret = sha_finish_walk(&req_ctx, cstate, false); 453ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (req_ctx) { 454ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen spin_lock(&cstate->work_lock); 455ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen list_del(&req_ctx->waiter); 456ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen spin_unlock(&cstate->work_lock); 457ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 458ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen req = cast_mcryptd_ctx_to_req(req_ctx); 459ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (irqs_disabled()) 460ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx->complete(&req->base, ret); 461ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen else { 462ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen local_bh_disable(); 463ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx->complete(&req->base, ret); 464ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen local_bh_enable(); 465ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 466ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 467ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha_ctx = sha1_ctx_mgr_get_comp_ctx(cstate->mgr); 468ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 469ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 470ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return 0; 471ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 472ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 473ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic void sha1_mb_add_list(struct mcryptd_hash_request_ctx *rctx, 474ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_alg_cstate *cstate) 475ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 476ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen unsigned long next_flush; 477ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL); 478ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 479ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* initialize tag */ 480ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx->tag.arrival = jiffies; /* tag the arrival time */ 481ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx->tag.seq_num = cstate->next_seq_num++; 482ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen next_flush = rctx->tag.arrival + delay; 483ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx->tag.expire = next_flush; 484ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 485ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen spin_lock(&cstate->work_lock); 486ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen list_add_tail(&rctx->waiter, &cstate->work_list); 487ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen spin_unlock(&cstate->work_lock); 488ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 489ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen mcryptd_arm_flusher(cstate, delay); 490ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 491ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 492ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic int sha1_mb_update(struct shash_desc *desc, const u8 *data, 493ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen unsigned int len) 494ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 495ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_hash_request_ctx *rctx = 496ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen container_of(desc, struct mcryptd_hash_request_ctx, desc); 497ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_alg_cstate *cstate = 498ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen this_cpu_ptr(sha1_mb_alg_state.alg_cstate); 499ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 500ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); 501ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct sha1_hash_ctx *sha_ctx; 502ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen int ret = 0, nbytes; 503ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 504ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 505ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* sanity check */ 506ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (rctx->tag.cpu != smp_processor_id()) { 507ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen pr_err("mcryptd error: cpu clash\n"); 508ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen goto done; 509ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 510ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 511ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* need to init context */ 512ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen req_ctx_init(rctx, desc); 513ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 514ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen nbytes = crypto_ahash_walk_first(req, &rctx->walk); 515ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 516ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (nbytes < 0) { 517ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ret = nbytes; 518ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen goto done; 519ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 520ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 521ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (crypto_ahash_walk_last(&rctx->walk)) 522ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx->flag |= HASH_DONE; 523ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 524ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* submit */ 525ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha_ctx = (struct sha1_hash_ctx *) shash_desc_ctx(desc); 526ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha1_mb_add_list(rctx, cstate); 527ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen kernel_fpu_begin(); 528ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, nbytes, HASH_UPDATE); 529ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen kernel_fpu_end(); 530ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 531ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* check if anything is returned */ 532ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (!sha_ctx) 533ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return -EINPROGRESS; 534ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 535ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (sha_ctx->error) { 536ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ret = sha_ctx->error; 537ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx = cast_hash_to_mcryptd_ctx(sha_ctx); 538ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen goto done; 539ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 540ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 541ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx = cast_hash_to_mcryptd_ctx(sha_ctx); 542ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ret = sha_finish_walk(&rctx, cstate, false); 543ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 544ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (!rctx) 545ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return -EINPROGRESS; 546ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chendone: 547ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha_complete_job(rctx, cstate, ret); 548ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return ret; 549ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 550ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 551ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic int sha1_mb_finup(struct shash_desc *desc, const u8 *data, 552ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen unsigned int len, u8 *out) 553ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 554ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_hash_request_ctx *rctx = 555ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen container_of(desc, struct mcryptd_hash_request_ctx, desc); 556ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_alg_cstate *cstate = 557ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen this_cpu_ptr(sha1_mb_alg_state.alg_cstate); 558ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 559ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); 560ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct sha1_hash_ctx *sha_ctx; 561ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen int ret = 0, flag = HASH_UPDATE, nbytes; 562ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 563ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* sanity check */ 564ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (rctx->tag.cpu != smp_processor_id()) { 565ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen pr_err("mcryptd error: cpu clash\n"); 566ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen goto done; 567ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 568ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 569ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* need to init context */ 570ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen req_ctx_init(rctx, desc); 571ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 572ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen nbytes = crypto_ahash_walk_first(req, &rctx->walk); 573ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 574ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (nbytes < 0) { 575ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ret = nbytes; 576ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen goto done; 577ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 578ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 579ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (crypto_ahash_walk_last(&rctx->walk)) { 580ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx->flag |= HASH_DONE; 581ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen flag = HASH_LAST; 582ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 583ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx->out = out; 584ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 585ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* submit */ 586ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx->flag |= HASH_FINAL; 587ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha_ctx = (struct sha1_hash_ctx *) shash_desc_ctx(desc); 588ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha1_mb_add_list(rctx, cstate); 589ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 590ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen kernel_fpu_begin(); 591ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, nbytes, flag); 592ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen kernel_fpu_end(); 593ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 594ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* check if anything is returned */ 595ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (!sha_ctx) 596ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return -EINPROGRESS; 597ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 598ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (sha_ctx->error) { 599ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ret = sha_ctx->error; 600ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen goto done; 601ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 602ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 603ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx = cast_hash_to_mcryptd_ctx(sha_ctx); 604ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ret = sha_finish_walk(&rctx, cstate, false); 605ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (!rctx) 606ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return -EINPROGRESS; 607ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chendone: 608ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha_complete_job(rctx, cstate, ret); 609ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return ret; 610ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 611ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 612ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic int sha1_mb_final(struct shash_desc *desc, u8 *out) 613ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 614ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_hash_request_ctx *rctx = 615ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen container_of(desc, struct mcryptd_hash_request_ctx, desc); 616ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_alg_cstate *cstate = 617ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen this_cpu_ptr(sha1_mb_alg_state.alg_cstate); 618ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 619ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct sha1_hash_ctx *sha_ctx; 620ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen int ret = 0; 621ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen u8 data; 622ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 623ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* sanity check */ 624ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (rctx->tag.cpu != smp_processor_id()) { 625ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen pr_err("mcryptd error: cpu clash\n"); 626ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen goto done; 627ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 628ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 629ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* need to init context */ 630ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen req_ctx_init(rctx, desc); 631ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 632ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx->out = out; 633ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx->flag |= HASH_DONE | HASH_FINAL; 634ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 635ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha_ctx = (struct sha1_hash_ctx *) shash_desc_ctx(desc); 636ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* flag HASH_FINAL and 0 data size */ 637ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha1_mb_add_list(rctx, cstate); 638ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen kernel_fpu_begin(); 639ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, &data, 0, HASH_LAST); 640ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen kernel_fpu_end(); 641ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 642ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* check if anything is returned */ 643ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (!sha_ctx) 644ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return -EINPROGRESS; 645ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 646ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (sha_ctx->error) { 647ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ret = sha_ctx->error; 648ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx = cast_hash_to_mcryptd_ctx(sha_ctx); 649ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen goto done; 650ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 651ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 652ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx = cast_hash_to_mcryptd_ctx(sha_ctx); 653ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ret = sha_finish_walk(&rctx, cstate, false); 654ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (!rctx) 655ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return -EINPROGRESS; 656ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chendone: 657ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha_complete_job(rctx, cstate, ret); 658ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return ret; 659ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 660ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 661ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic int sha1_mb_export(struct shash_desc *desc, void *out) 662ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 663ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct sha1_hash_ctx *sctx = shash_desc_ctx(desc); 664ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 665ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen memcpy(out, sctx, sizeof(*sctx)); 666ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 667ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return 0; 668ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 669ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 670ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic int sha1_mb_import(struct shash_desc *desc, const void *in) 671ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 672ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct sha1_hash_ctx *sctx = shash_desc_ctx(desc); 673ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 674ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen memcpy(sctx, in, sizeof(*sctx)); 675ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 676ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return 0; 677ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 678ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 679ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 680ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic struct shash_alg sha1_mb_shash_alg = { 681ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .digestsize = SHA1_DIGEST_SIZE, 682ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .init = sha1_mb_init, 683ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .update = sha1_mb_update, 684ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .final = sha1_mb_final, 685ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .finup = sha1_mb_finup, 686ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .export = sha1_mb_export, 687ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .import = sha1_mb_import, 688ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .descsize = sizeof(struct sha1_hash_ctx), 689ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .statesize = sizeof(struct sha1_hash_ctx), 690ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .base = { 691ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .cra_name = "__sha1-mb", 692ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .cra_driver_name = "__intel_sha1-mb", 693ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .cra_priority = 100, 694ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* 695ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * use ASYNC flag as some buffers in multi-buffer 696ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen * algo may not have completed before hashing thread sleep 697ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen */ 698ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .cra_flags = CRYPTO_ALG_TYPE_SHASH | CRYPTO_ALG_ASYNC, 699ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .cra_blocksize = SHA1_BLOCK_SIZE, 700ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .cra_module = THIS_MODULE, 701ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .cra_list = LIST_HEAD_INIT(sha1_mb_shash_alg.base.cra_list), 702ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 703ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen}; 704ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 705ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic int sha1_mb_async_init(struct ahash_request *req) 706ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 707ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 708ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); 709ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct ahash_request *mcryptd_req = ahash_request_ctx(req); 710ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; 711ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 712ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen memcpy(mcryptd_req, req, sizeof(*req)); 713ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); 714ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return crypto_ahash_init(mcryptd_req); 715ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 716ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 717ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic int sha1_mb_async_update(struct ahash_request *req) 718ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 719ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct ahash_request *mcryptd_req = ahash_request_ctx(req); 720ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 721ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 722ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); 723ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; 724ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 725ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen memcpy(mcryptd_req, req, sizeof(*req)); 726ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); 727ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return crypto_ahash_update(mcryptd_req); 728ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 729ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 730ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic int sha1_mb_async_finup(struct ahash_request *req) 731ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 732ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct ahash_request *mcryptd_req = ahash_request_ctx(req); 733ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 734ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 735ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); 736ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; 737ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 738ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen memcpy(mcryptd_req, req, sizeof(*req)); 739ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); 740ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return crypto_ahash_finup(mcryptd_req); 741ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 742ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 743ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic int sha1_mb_async_final(struct ahash_request *req) 744ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 745ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct ahash_request *mcryptd_req = ahash_request_ctx(req); 746ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 747ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 748ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); 749ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; 750ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 751ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen memcpy(mcryptd_req, req, sizeof(*req)); 752ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); 753ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return crypto_ahash_final(mcryptd_req); 754ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 755ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 7564c1948fc471d444b15dfaca73ea0c9b77c2d5505Fengguang Wustatic int sha1_mb_async_digest(struct ahash_request *req) 757ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 758ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 759ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); 760ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct ahash_request *mcryptd_req = ahash_request_ctx(req); 761ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; 762ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 763ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen memcpy(mcryptd_req, req, sizeof(*req)); 764ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); 765ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return crypto_ahash_digest(mcryptd_req); 766ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 767ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 768ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic int sha1_mb_async_init_tfm(struct crypto_tfm *tfm) 769ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 770ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_ahash *mcryptd_tfm; 771ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm); 772ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_hash_ctx *mctx; 773ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 774ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha1-mb", 0, 0); 775ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (IS_ERR(mcryptd_tfm)) 776ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return PTR_ERR(mcryptd_tfm); 777ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen mctx = crypto_ahash_ctx(&mcryptd_tfm->base); 778ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen mctx->alg_state = &sha1_mb_alg_state; 779ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen ctx->mcryptd_tfm = mcryptd_tfm; 780ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 781ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sizeof(struct ahash_request) + 782ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen crypto_ahash_reqsize(&mcryptd_tfm->base)); 783ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 784ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return 0; 785ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 786ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 787ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic void sha1_mb_async_exit_tfm(struct crypto_tfm *tfm) 788ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 789ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm); 790ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 791ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen mcryptd_free_ahash(ctx->mcryptd_tfm); 792ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 793ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 794ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic struct ahash_alg sha1_mb_async_alg = { 795ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .init = sha1_mb_async_init, 796ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .update = sha1_mb_async_update, 797ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .final = sha1_mb_async_final, 798ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .finup = sha1_mb_async_finup, 799ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .digest = sha1_mb_async_digest, 800ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .halg = { 801ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .digestsize = SHA1_DIGEST_SIZE, 802ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .base = { 803ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .cra_name = "sha1", 804ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .cra_driver_name = "sha1_mb", 805ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .cra_priority = 200, 806ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, 807ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .cra_blocksize = SHA1_BLOCK_SIZE, 808ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .cra_type = &crypto_ahash_type, 809ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .cra_module = THIS_MODULE, 810ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .cra_list = LIST_HEAD_INIT(sha1_mb_async_alg.halg.base.cra_list), 811ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .cra_init = sha1_mb_async_init_tfm, 812ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .cra_exit = sha1_mb_async_exit_tfm, 813ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .cra_ctxsize = sizeof(struct sha1_mb_ctx), 814ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen .cra_alignmask = 0, 815ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen }, 816ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen }, 817ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen}; 818ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 8194c1948fc471d444b15dfaca73ea0c9b77c2d5505Fengguang Wustatic unsigned long sha1_mb_flusher(struct mcryptd_alg_cstate *cstate) 820ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 821ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_hash_request_ctx *rctx; 822ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen unsigned long cur_time; 823ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen unsigned long next_flush = 0; 824ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct sha1_hash_ctx *sha_ctx; 825ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 826ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 827ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen cur_time = jiffies; 828ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 829ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen while (!list_empty(&cstate->work_list)) { 830ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx = list_entry(cstate->work_list.next, 831ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_hash_request_ctx, waiter); 832ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if time_before(cur_time, rctx->tag.expire) 833ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen break; 834ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen kernel_fpu_begin(); 835ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha_ctx = (struct sha1_hash_ctx *) sha1_ctx_mgr_flush(cstate->mgr); 836ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen kernel_fpu_end(); 837ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (!sha_ctx) { 838ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen pr_err("sha1_mb error: nothing got flushed for non-empty list\n"); 839ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen break; 840ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 841ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx = cast_hash_to_mcryptd_ctx(sha_ctx); 842ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha_finish_walk(&rctx, cstate, true); 843ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha_complete_job(rctx, cstate, 0); 844ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 845ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 846ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (!list_empty(&cstate->work_list)) { 847ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen rctx = list_entry(cstate->work_list.next, 848ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_hash_request_ctx, waiter); 849ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* get the hash context and then flush time */ 850ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen next_flush = rctx->tag.expire; 851ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen mcryptd_arm_flusher(cstate, get_delay(next_flush)); 852ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 853ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return next_flush; 854ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 855ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 856ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic int __init sha1_mb_mod_init(void) 857ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 858ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 859ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen int cpu; 860ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen int err; 861ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_alg_cstate *cpu_state; 862ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 863ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* check for dependent cpu features */ 864ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (!boot_cpu_has(X86_FEATURE_AVX2) || 865ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen !boot_cpu_has(X86_FEATURE_BMI2)) 866ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return -ENODEV; 867ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 868ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen /* initialize multibuffer structures */ 869ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha1_mb_alg_state.alg_cstate = alloc_percpu(struct mcryptd_alg_cstate); 870ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 871ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha1_job_mgr_init = sha1_mb_mgr_init_avx2; 872ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha1_job_mgr_submit = sha1_mb_mgr_submit_avx2; 873ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha1_job_mgr_flush = sha1_mb_mgr_flush_avx2; 874ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha1_job_mgr_get_comp_job = sha1_mb_mgr_get_comp_job_avx2; 875ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 876ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (!sha1_mb_alg_state.alg_cstate) 877ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return -ENOMEM; 878ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen for_each_possible_cpu(cpu) { 879ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu); 880ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen cpu_state->next_flush = 0; 881ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen cpu_state->next_seq_num = 0; 882ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen cpu_state->flusher_engaged = false; 883ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen INIT_DELAYED_WORK(&cpu_state->flush, mcryptd_flusher); 884ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen cpu_state->cpu = cpu; 885ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen cpu_state->alg_state = &sha1_mb_alg_state; 886ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen cpu_state->mgr = (struct sha1_ctx_mgr *) kzalloc(sizeof(struct sha1_ctx_mgr), GFP_KERNEL); 887ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (!cpu_state->mgr) 888ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen goto err2; 889ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha1_ctx_mgr_init(cpu_state->mgr); 890ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen INIT_LIST_HEAD(&cpu_state->work_list); 891ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen spin_lock_init(&cpu_state->work_lock); 892ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 893ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen sha1_mb_alg_state.flusher = &sha1_mb_flusher; 894ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 895ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen err = crypto_register_shash(&sha1_mb_shash_alg); 896ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (err) 897ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen goto err2; 898ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen err = crypto_register_ahash(&sha1_mb_async_alg); 899ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen if (err) 900ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen goto err1; 901ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 902ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 903ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return 0; 904ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenerr1: 905ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen crypto_unregister_shash(&sha1_mb_shash_alg); 906ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenerr2: 907ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen for_each_possible_cpu(cpu) { 908ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu); 909ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen kfree(cpu_state->mgr); 910ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 911ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen free_percpu(sha1_mb_alg_state.alg_cstate); 912ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen return -ENODEV; 913ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 914ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 915ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenstatic void __exit sha1_mb_mod_fini(void) 916ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen{ 917ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen int cpu; 918ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen struct mcryptd_alg_cstate *cpu_state; 919ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 920ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen crypto_unregister_ahash(&sha1_mb_async_alg); 921ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen crypto_unregister_shash(&sha1_mb_shash_alg); 922ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen for_each_possible_cpu(cpu) { 923ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu); 924ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen kfree(cpu_state->mgr); 925ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen } 926ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen free_percpu(sha1_mb_alg_state.alg_cstate); 927ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen} 928ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 929ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenmodule_init(sha1_mb_mod_init); 930ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chenmodule_exit(sha1_mb_mod_fini); 931ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 932ad61e042e9151b55b393d5875e467e7fe0c7470cTim ChenMODULE_LICENSE("GPL"); 933ad61e042e9151b55b393d5875e467e7fe0c7470cTim ChenMODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, multi buffer accelerated"); 934ad61e042e9151b55b393d5875e467e7fe0c7470cTim Chen 935ad61e042e9151b55b393d5875e467e7fe0c7470cTim ChenMODULE_ALIAS("sha1"); 936