rsa_pmeth.c revision 392aa7cc7d2b122614c5393c3e357da07fd07af3
1afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg/* crypto/rsa/rsa_pmeth.c */ 2afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg/* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL 3559aec47015b741e045d57362f7732b3a04f9450Brian Paul * project 2006. 45e3bc0c2a2bcdf59949410f94c9b705fc1281ce8Jouk Jansen */ 5559aec47015b741e045d57362f7732b3a04f9450Brian Paul/* ==================================================================== 6559aec47015b741e045d57362f7732b3a04f9450Brian Paul * Copyright (c) 2006 The OpenSSL Project. All rights reserved. 75e3bc0c2a2bcdf59949410f94c9b705fc1281ce8Jouk Jansen * 8afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * Redistribution and use in source and binary forms, with or without 9afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * modification, are permitted provided that the following conditions 10afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * are met: 11afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * 12afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * 1. Redistributions of source code must retain the above copyright 13afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * notice, this list of conditions and the following disclaimer. 145e3bc0c2a2bcdf59949410f94c9b705fc1281ce8Jouk Jansen * 15afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * 2. Redistributions in binary form must reproduce the above copyright 16afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * notice, this list of conditions and the following disclaimer in 175e3bc0c2a2bcdf59949410f94c9b705fc1281ce8Jouk Jansen * the documentation and/or other materials provided with the 18afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * distribution. 19afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * 20afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * 3. All advertising materials mentioning features or use of this 21afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * software must display the following acknowledgment: 22afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * "This product includes software developed by the OpenSSL Project 23afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" 24afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * 25afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to 26559aec47015b741e045d57362f7732b3a04f9450Brian Paul * endorse or promote products derived from this software without 27559aec47015b741e045d57362f7732b3a04f9450Brian Paul * prior written permission. For written permission, please contact 28559aec47015b741e045d57362f7732b3a04f9450Brian Paul * licensing@OpenSSL.org. 29559aec47015b741e045d57362f7732b3a04f9450Brian Paul * 30559aec47015b741e045d57362f7732b3a04f9450Brian Paul * 5. Products derived from this software may not be called "OpenSSL" 31afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * nor may "OpenSSL" appear in their names without prior written 32fbd8f212c3866ec98c1d8c9d3db3ddb7e7c479a5Brian Paul * permission of the OpenSSL Project. 33c893a015d8a50a38cd3f727d99835e7e7e2ccea9Brian Paul * 34afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * 6. Redistributions of any form whatsoever must retain the following 35afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * acknowledgment: 36afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * "This product includes software developed by the OpenSSL Project 37afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" 385e3bc0c2a2bcdf59949410f94c9b705fc1281ce8Jouk Jansen * 39afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY 40afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 416dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 426dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR 43afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 44afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 45afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 46afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 47afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 48afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 49afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 50afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * OF THE POSSIBILITY OF SUCH DAMAGE. 51afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * ==================================================================== 52c40d1dd62dd9bcbb97128e37a75d991a8d3b2d8cKendall Bennett * 53fbd8f212c3866ec98c1d8c9d3db3ddb7e7c479a5Brian Paul * This product includes cryptographic software written by Eric Young 54afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg * (eay@cryptsoft.com). This product includes software written by Tim 55fbd8f212c3866ec98c1d8c9d3db3ddb7e7c479a5Brian Paul * Hudson (tjh@cryptsoft.com). 5622144ab7552f0799bcfca506bf4ffa7f70a06649Gareth Hughes * 57afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg */ 58afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg 5908836341788a9f9d638d9dc8328510ccd18ddeb5Brian Paul#include <stdio.h> 60afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg#include "cryptlib.h" 61afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg#include <openssl/asn1t.h> 62afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg#include <openssl/x509.h> 6308836341788a9f9d638d9dc8328510ccd18ddeb5Brian Paul#include <openssl/rsa.h> 64afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg#include <openssl/bn.h> 65afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg#include <openssl/evp.h> 66afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg#ifndef OPENSSL_NO_CMS 6708836341788a9f9d638d9dc8328510ccd18ddeb5Brian Paul#include <openssl/cms.h> 6822144ab7552f0799bcfca506bf4ffa7f70a06649Gareth Hughes#endif 69afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg#ifdef OPENSSL_FIPS 70afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg#include <openssl/fips.h> 71afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg#endif 72afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg#include "evp_locl.h" 73afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg#include "rsa_locl.h" 74cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell 75afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg/* RSA pkey context structure */ 76afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg 77cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwelltypedef struct 78afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 79afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg /* Key gen parameters */ 8022144ab7552f0799bcfca506bf4ffa7f70a06649Gareth Hughes int nbits; 81b6bcae5698df88f7730d40004ce7ce0462e97a20Brian Paul BIGNUM *pub_exp; 82afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg /* Keygen callback info */ 83afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg int gentmp[2]; 8422144ab7552f0799bcfca506bf4ffa7f70a06649Gareth Hughes /* RSA padding mode */ 8522144ab7552f0799bcfca506bf4ffa7f70a06649Gareth Hughes int pad_mode; 86cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell /* message digest */ 87afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg const EVP_MD *md; 88afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg /* message digest for MGF1 */ 8922144ab7552f0799bcfca506bf4ffa7f70a06649Gareth Hughes const EVP_MD *mgf1md; 90b6bcae5698df88f7730d40004ce7ce0462e97a20Brian Paul /* PSS/OAEP salt length */ 91cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell int saltlen; 92afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg /* Temp buffer */ 93afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg unsigned char *tbuf; 9408836341788a9f9d638d9dc8328510ccd18ddeb5Brian Paul } RSA_PKEY_CTX; 95cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell 96afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtgstatic int pkey_rsa_init(EVP_PKEY_CTX *ctx) 97afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 98cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell RSA_PKEY_CTX *rctx; 99cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell rctx = OPENSSL_malloc(sizeof(RSA_PKEY_CTX)); 100afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (!rctx) 101afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return 0; 102cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell rctx->nbits = 1024; 103afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg rctx->pub_exp = NULL; 104afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg rctx->pad_mode = RSA_PKCS1_PADDING; 105afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg rctx->md = NULL; 106c40d1dd62dd9bcbb97128e37a75d991a8d3b2d8cKendall Bennett rctx->mgf1md = NULL; 107fbd8f212c3866ec98c1d8c9d3db3ddb7e7c479a5Brian Paul rctx->tbuf = NULL; 108afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg 109fbd8f212c3866ec98c1d8c9d3db3ddb7e7c479a5Brian Paul rctx->saltlen = -2; 110cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell 111afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg ctx->data = rctx; 112afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg ctx->keygen_info = rctx->gentmp; 113cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell ctx->keygen_info_count = 2; 114afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg 115afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return 1; 116afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg } 117afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg 118afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtgstatic int pkey_rsa_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src) 119afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 120559aec47015b741e045d57362f7732b3a04f9450Brian Paul RSA_PKEY_CTX *dctx, *sctx; 121afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (!pkey_rsa_init(dst)) 122afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return 0; 123559aec47015b741e045d57362f7732b3a04f9450Brian Paul sctx = src->data; 124559aec47015b741e045d57362f7732b3a04f9450Brian Paul dctx = dst->data; 125559aec47015b741e045d57362f7732b3a04f9450Brian Paul dctx->nbits = sctx->nbits; 126559aec47015b741e045d57362f7732b3a04f9450Brian Paul if (sctx->pub_exp) 127559aec47015b741e045d57362f7732b3a04f9450Brian Paul { 128559aec47015b741e045d57362f7732b3a04f9450Brian Paul dctx->pub_exp = BN_dup(sctx->pub_exp); 129afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (!dctx->pub_exp) 1301bf9dfaf5dea61e3d33a69b0a549be54ef6d74dfKeith Whitwell return 0; 1311bf9dfaf5dea61e3d33a69b0a549be54ef6d74dfKeith Whitwell } 132cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell dctx->pad_mode = sctx->pad_mode; 1331bf9dfaf5dea61e3d33a69b0a549be54ef6d74dfKeith Whitwell dctx->md = sctx->md; 134afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return 1; 135cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell } 1361bf9dfaf5dea61e3d33a69b0a549be54ef6d74dfKeith Whitwell 137afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtgstatic int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk) 138cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell { 1391bf9dfaf5dea61e3d33a69b0a549be54ef6d74dfKeith Whitwell if (ctx->tbuf) 140afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return 1; 141cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell ctx->tbuf = OPENSSL_malloc(EVP_PKEY_size(pk->pkey)); 142afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (!ctx->tbuf) 143afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return 0; 144afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return 1; 145afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg } 146afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg 147cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwellstatic void pkey_rsa_cleanup(EVP_PKEY_CTX *ctx) 148afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 149afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg RSA_PKEY_CTX *rctx = ctx->data; 150afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (rctx) 151afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 152afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (rctx->pub_exp) 153afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg BN_free(rctx->pub_exp); 154afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (rctx->tbuf) 155559aec47015b741e045d57362f7732b3a04f9450Brian Paul OPENSSL_free(rctx->tbuf); 156afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg OPENSSL_free(rctx); 157afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg } 1586dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell } 1596dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell#ifdef OPENSSL_FIPS 1606dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell/* FIP checker. Return value indicates status of context parameters: 1616dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell * 1 : redirect to FIPS. 1626dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell * 0 : don't redirect to FIPS. 1636dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell * -1 : illegal operation in FIPS mode. 1646dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell */ 1656dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell 1666dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwellstatic int pkey_fips_check_ctx(EVP_PKEY_CTX *ctx) 1676dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell { 1686dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell RSA_PKEY_CTX *rctx = ctx->data; 1696dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell RSA *rsa = ctx->pkey->pkey.rsa; 1706dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell int rv = -1; 1716dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell if (!FIPS_mode()) 1726dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell return 0; 1736dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell if (rsa->flags & RSA_FLAG_NON_FIPS_ALLOW) 174afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg rv = 0; 175c40d1dd62dd9bcbb97128e37a75d991a8d3b2d8cKendall Bennett if (!(rsa->meth->flags & RSA_FLAG_FIPS_METHOD) && rv) 176fbd8f212c3866ec98c1d8c9d3db3ddb7e7c479a5Brian Paul return -1; 177afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (rctx->md && !(rctx->md->flags & EVP_MD_FLAG_FIPS)) 178fbd8f212c3866ec98c1d8c9d3db3ddb7e7c479a5Brian Paul return rv; 179cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell if (rctx->mgf1md && !(rctx->mgf1md->flags & EVP_MD_FLAG_FIPS)) 180cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell return rv; 181afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return 1; 18208836341788a9f9d638d9dc8328510ccd18ddeb5Brian Paul } 183cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell#endif 184afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg 185cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwellstatic int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen, 1866dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell const unsigned char *tbs, size_t tbslen) 187afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 188afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg int ret; 189afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg RSA_PKEY_CTX *rctx = ctx->data; 190afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg RSA *rsa = ctx->pkey->pkey.rsa; 191afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg 192afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg#ifdef OPENSSL_FIPS 193afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg ret = pkey_fips_check_ctx(ctx); 194afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (ret < 0) 195afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 1966dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell RSAerr(RSA_F_PKEY_RSA_SIGN, RSA_R_OPERATION_NOT_ALLOWED_IN_FIPS_MODE); 1976dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell return -1; 1986dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell } 1996dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell#endif 2006dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell 2016dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell if (rctx->md) 2026dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell { 2036dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell if (tbslen != (size_t)EVP_MD_size(rctx->md)) 2046dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell { 205afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg RSAerr(RSA_F_PKEY_RSA_SIGN, 206afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg RSA_R_INVALID_DIGEST_LENGTH); 207afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return -1; 208afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg } 209afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg#ifdef OPENSSL_FIPS 210afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (ret > 0) 211afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 2126dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell unsigned int slen; 2136dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell ret = FIPS_rsa_sign_digest(rsa, tbs, tbslen, rctx->md, 2146dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell rctx->pad_mode, 2156dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell rctx->saltlen, 2166dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell rctx->mgf1md, 2176dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell sig, &slen); 2186dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell if (ret > 0) 2196dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell *siglen = slen; 2206dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell else 221559aec47015b741e045d57362f7732b3a04f9450Brian Paul *siglen = 0; 222559aec47015b741e045d57362f7732b3a04f9450Brian Paul return ret; 223afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg } 224afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg#endif 225afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg 226afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (EVP_MD_type(rctx->md) == NID_mdc2) 227afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 228afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg unsigned int sltmp; 229afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (rctx->pad_mode != RSA_PKCS1_PADDING) 230afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return -1; 231afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg ret = RSA_sign_ASN1_OCTET_STRING(NID_mdc2, 232afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg tbs, tbslen, sig, &sltmp, rsa); 233afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg 2346dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell if (ret <= 0) 2356dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell return ret; 2366dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell ret = sltmp; 2376dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell } 2386dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell else if (rctx->pad_mode == RSA_X931_PADDING) 2396dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell { 2406dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell if (!setup_tbuf(rctx, ctx)) 2416dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell return -1; 2426dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell memcpy(rctx->tbuf, tbs, tbslen); 2436dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell rctx->tbuf[tbslen] = 2446dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell RSA_X931_hash_id(EVP_MD_type(rctx->md)); 245559aec47015b741e045d57362f7732b3a04f9450Brian Paul ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf, 246559aec47015b741e045d57362f7732b3a04f9450Brian Paul sig, rsa, RSA_X931_PADDING); 247afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg } 248afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg else if (rctx->pad_mode == RSA_PKCS1_PADDING) 249afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 250afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg unsigned int sltmp; 251afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg ret = RSA_sign(EVP_MD_type(rctx->md), 252afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg tbs, tbslen, sig, &sltmp, rsa); 253afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (ret <= 0) 254afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return ret; 255afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg ret = sltmp; 256afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg } 257afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) 258afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 259afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (!setup_tbuf(rctx, ctx)) 260afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return -1; 2613be135f0000515f677c3ca927034caa2fc669e2eBrian Paul if (!RSA_padding_add_PKCS1_PSS_mgf1(rsa, 262afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg rctx->tbuf, tbs, 263afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg rctx->md, rctx->mgf1md, 264afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg rctx->saltlen)) 265afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return -1; 266afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf, 267afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg sig, rsa, RSA_NO_PADDING); 268afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg } 269afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg else 270afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return -1; 271afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg } 2726dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell else 2736dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa, 2746dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell rctx->pad_mode); 2756dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell if (ret < 0) 2766dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell return ret; 2776dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell *siglen = ret; 2786dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell return 1; 279c40d1dd62dd9bcbb97128e37a75d991a8d3b2d8cKendall Bennett } 280fbd8f212c3866ec98c1d8c9d3db3ddb7e7c479a5Brian Paul 281afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg 282fbd8f212c3866ec98c1d8c9d3db3ddb7e7c479a5Brian Paulstatic int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx, 283cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell unsigned char *rout, size_t *routlen, 284cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell const unsigned char *sig, size_t siglen) 285afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 2863be135f0000515f677c3ca927034caa2fc669e2eBrian Paul int ret; 287afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg RSA_PKEY_CTX *rctx = ctx->data; 288afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg 289afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (rctx->md) 290afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 291afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (rctx->pad_mode == RSA_X931_PADDING) 292afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 293afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (!setup_tbuf(rctx, ctx)) 294afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return -1; 295cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell ret = RSA_public_decrypt(siglen, sig, 296afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg rctx->tbuf, ctx->pkey->pkey.rsa, 297afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg RSA_X931_PADDING); 298afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (ret < 1) 2996dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell return 0; 3006dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell ret--; 3016dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell if (rctx->tbuf[ret] != 3026dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell RSA_X931_hash_id(EVP_MD_type(rctx->md))) 3036dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell { 3046dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER, 3056dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell RSA_R_ALGORITHM_MISMATCH); 3066dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell return 0; 3076dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell } 3086dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell if (ret != EVP_MD_size(rctx->md)) 3096dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell { 310c40d1dd62dd9bcbb97128e37a75d991a8d3b2d8cKendall Bennett RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER, 311fbd8f212c3866ec98c1d8c9d3db3ddb7e7c479a5Brian Paul RSA_R_INVALID_DIGEST_LENGTH); 312afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return 0; 313fbd8f212c3866ec98c1d8c9d3db3ddb7e7c479a5Brian Paul } 314cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell if (rout) 315cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell memcpy(rout, rctx->tbuf, ret); 3163be135f0000515f677c3ca927034caa2fc669e2eBrian Paul } 317afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg else if (rctx->pad_mode == RSA_PKCS1_PADDING) 318afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 3193be135f0000515f677c3ca927034caa2fc669e2eBrian Paul size_t sltmp; 32008836341788a9f9d638d9dc8328510ccd18ddeb5Brian Paul ret = int_rsa_verify(EVP_MD_type(rctx->md), 321afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg NULL, 0, rout, &sltmp, 322afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg sig, siglen, ctx->pkey->pkey.rsa); 323cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell if (ret <= 0) 324cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell return 0; 325cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell ret = sltmp; 326afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg } 327afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg else 328afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return -1; 3293be135f0000515f677c3ca927034caa2fc669e2eBrian Paul } 330afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg else 331afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa, 332afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg rctx->pad_mode); 333afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (ret < 0) 334afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return ret; 335afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg *routlen = ret; 336afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return 1; 337afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg } 3386dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell 3396dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwellstatic int pkey_rsa_verify(EVP_PKEY_CTX *ctx, 3406dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell const unsigned char *sig, size_t siglen, 3416dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell const unsigned char *tbs, size_t tbslen) 3426dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell { 3436dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell RSA_PKEY_CTX *rctx = ctx->data; 3446dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell RSA *rsa = ctx->pkey->pkey.rsa; 3456dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell size_t rslen; 3466dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell#ifdef OPENSSL_FIPS 3476dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell int rv; 3486dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell rv = pkey_fips_check_ctx(ctx); 349c40d1dd62dd9bcbb97128e37a75d991a8d3b2d8cKendall Bennett if (rv < 0) 350fbd8f212c3866ec98c1d8c9d3db3ddb7e7c479a5Brian Paul { 351afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg RSAerr(RSA_F_PKEY_RSA_VERIFY, RSA_R_OPERATION_NOT_ALLOWED_IN_FIPS_MODE); 352fbd8f212c3866ec98c1d8c9d3db3ddb7e7c479a5Brian Paul return -1; 353cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell } 354cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell#endif 3553be135f0000515f677c3ca927034caa2fc669e2eBrian Paul if (rctx->md) 356afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 357afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg#ifdef OPENSSL_FIPS 358cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell if (rv > 0) 359cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell { 360afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return FIPS_rsa_verify_digest(rsa, 361afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg tbs, tbslen, 362afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg rctx->md, 363cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell rctx->pad_mode, 36408836341788a9f9d638d9dc8328510ccd18ddeb5Brian Paul rctx->saltlen, 365afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg rctx->mgf1md, 366cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell sig, siglen); 367cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell 368afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg } 369afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg#endif 370afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (rctx->pad_mode == RSA_PKCS1_PADDING) 3716dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen, 3726dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell sig, siglen, rsa); 3736dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell if (rctx->pad_mode == RSA_X931_PADDING) 3746dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell { 3756dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell if (pkey_rsa_verifyrecover(ctx, NULL, &rslen, 3766dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell sig, siglen) <= 0) 3776dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell return 0; 3786dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell } 3796dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) 380c40d1dd62dd9bcbb97128e37a75d991a8d3b2d8cKendall Bennett { 381fbd8f212c3866ec98c1d8c9d3db3ddb7e7c479a5Brian Paul int ret; 382afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (!setup_tbuf(rctx, ctx)) 383fbd8f212c3866ec98c1d8c9d3db3ddb7e7c479a5Brian Paul return -1; 384cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell ret = RSA_public_decrypt(siglen, sig, rctx->tbuf, 385cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell rsa, RSA_NO_PADDING); 3863be135f0000515f677c3ca927034caa2fc669e2eBrian Paul if (ret <= 0) 387afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return 0; 388afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg ret = RSA_verify_PKCS1_PSS_mgf1(rsa, tbs, 389cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell rctx->md, rctx->mgf1md, 390cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell rctx->tbuf, rctx->saltlen); 391afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (ret <= 0) 392afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return 0; 393afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return 1; 394cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell } 39508836341788a9f9d638d9dc8328510ccd18ddeb5Brian Paul else 396afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return -1; 397cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell } 398cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell else 399afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 400afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (!setup_tbuf(rctx, ctx)) 4016dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell return -1; 402afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf, 403afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg rsa, rctx->pad_mode); 404afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (rslen == 0) 4056dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell return 0; 4066dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell } 407afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg 4086dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell if ((rslen != tbslen) || memcmp(tbs, rctx->tbuf, rslen)) 4096dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell return 0; 4106dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell 4116dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell return 1; 4126dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell 4136dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell } 4146dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell 4156dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell 4166dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwellstatic int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, 4176dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell unsigned char *out, size_t *outlen, 4186dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell const unsigned char *in, size_t inlen) 4196dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell { 4206dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell int ret; 4216dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell RSA_PKEY_CTX *rctx = ctx->data; 422afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa, 423c40d1dd62dd9bcbb97128e37a75d991a8d3b2d8cKendall Bennett rctx->pad_mode); 424fbd8f212c3866ec98c1d8c9d3db3ddb7e7c479a5Brian Paul if (ret < 0) 425afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return ret; 426fbd8f212c3866ec98c1d8c9d3db3ddb7e7c479a5Brian Paul *outlen = ret; 427afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return 1; 428cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell } 429afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg 430afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtgstatic int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, 4319d47f3d6b8e8054602a637fad70d4769427df8b4Brian Paul unsigned char *out, size_t *outlen, 432afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg const unsigned char *in, size_t inlen) 433cab974cf6c2dbfbf5dd5d291e1aae0f8eeb34290Keith Whitwell { 434afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg int ret; 435afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg RSA_PKEY_CTX *rctx = ctx->data; 436afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa, 437afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg rctx->pad_mode); 438afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (ret < 0) 439afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return ret; 440afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg *outlen = ret; 441afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return 1; 442afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg } 443afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg 444afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtgstatic int check_padding_md(const EVP_MD *md, int padding) 445afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 446b1394fa92aaaf859ce9efc8b5fc194397921320cBrian Paul if (!md) 447afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return 1; 448afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg 449afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (padding == RSA_NO_PADDING) 450afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 451afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_PADDING_MODE); 452afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return 0; 453afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg } 454afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg 455afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (padding == RSA_X931_PADDING) 456afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 4576dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell if (RSA_X931_hash_id(EVP_MD_type(md)) == -1) 458afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 459afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg RSAerr(RSA_F_CHECK_PADDING_MD, 460afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg RSA_R_INVALID_X931_DIGEST); 461afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return 0; 462afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg } 463afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return 1; 464afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg } 465afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg 466afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return 1; 467afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg } 4686dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell 469afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg 47008836341788a9f9d638d9dc8328510ccd18ddeb5Brian Paulstatic int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2) 471afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 472afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg RSA_PKEY_CTX *rctx = ctx->data; 473afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg switch (type) 474afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 475afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg case EVP_PKEY_CTRL_RSA_PADDING: 476afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if ((p1 >= RSA_PKCS1_PADDING) && (p1 <= RSA_PKCS1_PSS_PADDING)) 477afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 478afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (!check_padding_md(rctx->md, p1)) 479afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return 0; 48008836341788a9f9d638d9dc8328510ccd18ddeb5Brian Paul if (p1 == RSA_PKCS1_PSS_PADDING) 481afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 482afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (!(ctx->operation & 4836dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY))) 484afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg goto bad_pad; 485afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (!rctx->md) 486afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg rctx->md = EVP_sha1(); 48708836341788a9f9d638d9dc8328510ccd18ddeb5Brian Paul } 488afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg if (p1 == RSA_PKCS1_OAEP_PADDING) 489afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg { 4906dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT)) 491afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg goto bad_pad; 49208836341788a9f9d638d9dc8328510ccd18ddeb5Brian Paul if (!rctx->md) 493afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg rctx->md = EVP_sha1(); 494afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg } 495afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg rctx->pad_mode = p1; 496afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return 1; 49722144ab7552f0799bcfca506bf4ffa7f70a06649Gareth Hughes } 498b980b2eeb62dc48101a7481d02d196c80b9da397Keith Whitwell bad_pad: 499b980b2eeb62dc48101a7481d02d196c80b9da397Keith Whitwell RSAerr(RSA_F_PKEY_RSA_CTRL, 500afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE); 501afb833d4e89c312460a4ab9ed6a7a8ca4ebbfe1cjtg return -2; 5026dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell 5036dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell case EVP_PKEY_CTRL_GET_RSA_PADDING: 5046dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell *(int *)p2 = rctx->pad_mode; 5056dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell return 1; 5066dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell 5076dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell case EVP_PKEY_CTRL_RSA_PSS_SALTLEN: 5086dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell case EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN: 5096dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING) 5106dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell { 5116dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN); 5126dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell return -2; 5136dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell } 5146dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell if (type == EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN) 5156dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell *(int *)p2 = rctx->saltlen; 5166dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell else 5176dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell { 5186dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell if (p1 < -2) 5196dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell return -2; 5206dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell rctx->saltlen = p1; 5216dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell } 5226dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell return 1; 5236dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell 5246dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell case EVP_PKEY_CTRL_RSA_KEYGEN_BITS: 5256dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell if (p1 < 256) 5266dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell { 5276dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_KEYBITS); 5286dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell return -2; 5296dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell } 5306dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell rctx->nbits = p1; 5316dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell return 1; 5326dc85575000127630489b407c50a4b3ea87c9acbKeith Whitwell 533 case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP: 534 if (!p2) 535 return -2; 536 rctx->pub_exp = p2; 537 return 1; 538 539 case EVP_PKEY_CTRL_MD: 540 if (!check_padding_md(p2, rctx->pad_mode)) 541 return 0; 542 rctx->md = p2; 543 return 1; 544 545 case EVP_PKEY_CTRL_RSA_MGF1_MD: 546 case EVP_PKEY_CTRL_GET_RSA_MGF1_MD: 547 if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING) 548 { 549 RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_MGF1_MD); 550 return -2; 551 } 552 if (type == EVP_PKEY_CTRL_GET_RSA_MGF1_MD) 553 { 554 if (rctx->mgf1md) 555 *(const EVP_MD **)p2 = rctx->mgf1md; 556 else 557 *(const EVP_MD **)p2 = rctx->md; 558 } 559 else 560 rctx->mgf1md = p2; 561 return 1; 562 563 case EVP_PKEY_CTRL_DIGESTINIT: 564 case EVP_PKEY_CTRL_PKCS7_ENCRYPT: 565 case EVP_PKEY_CTRL_PKCS7_DECRYPT: 566 case EVP_PKEY_CTRL_PKCS7_SIGN: 567 return 1; 568#ifndef OPENSSL_NO_CMS 569 case EVP_PKEY_CTRL_CMS_DECRYPT: 570 { 571 X509_ALGOR *alg = NULL; 572 ASN1_OBJECT *encalg = NULL; 573 if (p2) 574 CMS_RecipientInfo_ktri_get0_algs(p2, NULL, NULL, &alg); 575 if (alg) 576 X509_ALGOR_get0(&encalg, NULL, NULL, alg); 577 if (encalg && OBJ_obj2nid(encalg) == NID_rsaesOaep) 578 rctx->pad_mode = RSA_PKCS1_OAEP_PADDING; 579 } 580 case EVP_PKEY_CTRL_CMS_ENCRYPT: 581 case EVP_PKEY_CTRL_CMS_SIGN: 582 return 1; 583#endif 584 case EVP_PKEY_CTRL_PEER_KEY: 585 RSAerr(RSA_F_PKEY_RSA_CTRL, 586 RSA_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); 587 return -2; 588 589 default: 590 return -2; 591 592 } 593 } 594 595static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx, 596 const char *type, const char *value) 597 { 598 if (!value) 599 { 600 RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_VALUE_MISSING); 601 return 0; 602 } 603 if (!strcmp(type, "rsa_padding_mode")) 604 { 605 int pm; 606 if (!strcmp(value, "pkcs1")) 607 pm = RSA_PKCS1_PADDING; 608 else if (!strcmp(value, "sslv23")) 609 pm = RSA_SSLV23_PADDING; 610 else if (!strcmp(value, "none")) 611 pm = RSA_NO_PADDING; 612 else if (!strcmp(value, "oeap")) 613 pm = RSA_PKCS1_OAEP_PADDING; 614 else if (!strcmp(value, "x931")) 615 pm = RSA_X931_PADDING; 616 else if (!strcmp(value, "pss")) 617 pm = RSA_PKCS1_PSS_PADDING; 618 else 619 { 620 RSAerr(RSA_F_PKEY_RSA_CTRL_STR, 621 RSA_R_UNKNOWN_PADDING_TYPE); 622 return -2; 623 } 624 return EVP_PKEY_CTX_set_rsa_padding(ctx, pm); 625 } 626 627 if (!strcmp(type, "rsa_pss_saltlen")) 628 { 629 int saltlen; 630 saltlen = atoi(value); 631 return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen); 632 } 633 634 if (!strcmp(type, "rsa_keygen_bits")) 635 { 636 int nbits; 637 nbits = atoi(value); 638 return EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, nbits); 639 } 640 641 if (!strcmp(type, "rsa_keygen_pubexp")) 642 { 643 int ret; 644 BIGNUM *pubexp = NULL; 645 if (!BN_asc2bn(&pubexp, value)) 646 return 0; 647 ret = EVP_PKEY_CTX_set_rsa_keygen_pubexp(ctx, pubexp); 648 if (ret <= 0) 649 BN_free(pubexp); 650 return ret; 651 } 652 653 return -2; 654 } 655 656static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey) 657 { 658 RSA *rsa = NULL; 659 RSA_PKEY_CTX *rctx = ctx->data; 660 BN_GENCB *pcb, cb; 661 int ret; 662 if (!rctx->pub_exp) 663 { 664 rctx->pub_exp = BN_new(); 665 if (!rctx->pub_exp || !BN_set_word(rctx->pub_exp, RSA_F4)) 666 return 0; 667 } 668 rsa = RSA_new(); 669 if (!rsa) 670 return 0; 671 if (ctx->pkey_gencb) 672 { 673 pcb = &cb; 674 evp_pkey_set_cb_translate(pcb, ctx); 675 } 676 else 677 pcb = NULL; 678 ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb); 679 if (ret > 0) 680 EVP_PKEY_assign_RSA(pkey, rsa); 681 else 682 RSA_free(rsa); 683 return ret; 684 } 685 686const EVP_PKEY_METHOD rsa_pkey_meth = 687 { 688 EVP_PKEY_RSA, 689 EVP_PKEY_FLAG_AUTOARGLEN, 690 pkey_rsa_init, 691 pkey_rsa_copy, 692 pkey_rsa_cleanup, 693 694 0,0, 695 696 0, 697 pkey_rsa_keygen, 698 699 0, 700 pkey_rsa_sign, 701 702 0, 703 pkey_rsa_verify, 704 705 0, 706 pkey_rsa_verifyrecover, 707 708 709 0,0,0,0, 710 711 0, 712 pkey_rsa_encrypt, 713 714 0, 715 pkey_rsa_decrypt, 716 717 0,0, 718 719 pkey_rsa_ctrl, 720 pkey_rsa_ctrl_str 721 722 723 }; 724