generic-library/openssl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix to PKCS#7 routines so it can decrypt some oddball RC2 handling.

Browse Source
This commit is contained in:
Dr. Stephen Henson 1999-10-29 13:06:25 +00:00
parent 74400f7348
commit 9716a8f9f2
2 changed files with 34 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
CHANGES
View File

@ -4,6 +4,18 @@
Changes between 0.9.4 and 0.9.5 [xx XXX 1999] Changes between 0.9.4 and 0.9.5 [xx XXX 1999]
*) Hack to fix PKCS#7 decryption when used with some unorthodox RC2
handling. Most clients have the effective key size in bits equal to
the key length in bits: so a 40 bit RC2 key uses a 40 bit (5 byte) key.
A few however don't do this and instead use the size of the decrypted key
to determine the RC2 key length and the AlgorithmIdentifier to determine
the effective key length. In this case the effective key lenth can still
be 40 bits but the key length can be 168 bits for example. This is fixed
by manually forcing an RC2 key into the EVP_PKEY structure because the
EVP code can't currently handle unusual RC2 key sizes: it always assumes
the key length and effective key length are equal.
[Steve Henson]
*) Add a bunch of functions that should simplify the creation of *) Add a bunch of functions that should simplify the creation of
X509_NAME structures. Now you should be able to do: X509_NAME structures. Now you should be able to do:
X509_NAME_add_entry_by_txt(nm, "CN", MBSTRING_ASC, "Steve", -1, -1, 0); X509_NAME_add_entry_by_txt(nm, "CN", MBSTRING_ASC, "Steve", -1, -1, 0);

33
crypto/pkcs7/pk7_doit.c
View File

@ -251,7 +251,7 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
{ {
int i,j; int i,j;
BIO *out=NULL,*btmp=NULL,*etmp=NULL,*bio=NULL; BIO *out=NULL,*btmp=NULL,*etmp=NULL,*bio=NULL;
char *tmp=NULL; unsigned char *tmp=NULL;
X509_ALGOR *xa; X509_ALGOR *xa;
ASN1_OCTET_STRING *data_body=NULL; ASN1_OCTET_STRING *data_body=NULL;
const EVP_MD *evp_md; const EVP_MD *evp_md;
@ -262,6 +262,7 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
STACK_OF(PKCS7_RECIP_INFO) *rsk=NULL; STACK_OF(PKCS7_RECIP_INFO) *rsk=NULL;
X509_ALGOR *xalg=NULL; X509_ALGOR *xalg=NULL;
PKCS7_RECIP_INFO *ri=NULL; PKCS7_RECIP_INFO *ri=NULL;
char is_rc2 = 0;
/* EVP_PKEY *pkey; */ /* EVP_PKEY *pkey; */
#if 0 #if 0
X509_STORE_CTX s_ctx; X509_STORE_CTX s_ctx;
@ -306,6 +307,8 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
goto err; goto err;
} }
if(EVP_CIPHER_nid(evp_cipher) == NID_rc2_cbc) is_rc2 = 1;
/* We will be checking the signature */ /* We will be checking the signature */
if (md_sk != NULL) if (md_sk != NULL)
{ {
@ -375,17 +378,15 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
} }
jj=EVP_PKEY_size(pkey); jj=EVP_PKEY_size(pkey);
tmp=Malloc(jj+10); tmp=(unsigned char *)Malloc(jj+10);
if (tmp == NULL) if (tmp == NULL)
{ {
PKCS7err(PKCS7_F_PKCS7_DATADECODE,ERR_R_MALLOC_FAILURE); PKCS7err(PKCS7_F_PKCS7_DATADECODE,ERR_R_MALLOC_FAILURE);
goto err; goto err;
} }
jj=EVP_PKEY_decrypt((unsigned char *)tmp, jj=EVP_PKEY_decrypt(tmp, M_ASN1_STRING_data(ri->enc_key),
M_ASN1_STRING_data(ri->enc_key), M_ASN1_STRING_length(ri->enc_key), pkey);
M_ASN1_STRING_length(ri->enc_key),
pkey);
if (jj <= 0) if (jj <= 0)
{ {
PKCS7err(PKCS7_F_PKCS7_DATADECODE,ERR_R_EVP_LIB); PKCS7err(PKCS7_F_PKCS7_DATADECODE,ERR_R_EVP_LIB);
@ -398,13 +399,23 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
if (EVP_CIPHER_asn1_to_param(evp_ctx,enc_alg->parameter) < 0) if (EVP_CIPHER_asn1_to_param(evp_ctx,enc_alg->parameter) < 0)
return(NULL); return(NULL);
if (jj != EVP_CIPHER_CTX_key_length(evp_ctx)) if (jj != EVP_CIPHER_CTX_key_length(evp_ctx)) {
{ /* HACK: some S/MIME clients don't use the same key
PKCS7err(PKCS7_F_PKCS7_DATADECODE, * and effective key length. The key length is
* determined by the size of the decrypted RSA key.
* So we hack things to manually set the RC2 key
* because we currently can't do this with the EVP
* interface.
*/
if(is_rc2) RC2_set_key(&(evp_ctx->c.rc2_ks),jj, tmp,
EVP_CIPHER_CTX_key_length(evp_ctx)*8);
else {
PKCS7err(PKCS7_F_PKCS7_DATADECODE,
PKCS7_R_DECRYPTED_KEY_IS_WRONG_LENGTH); PKCS7_R_DECRYPTED_KEY_IS_WRONG_LENGTH);
goto err; goto err;
} }
EVP_CipherInit(evp_ctx,NULL,(unsigned char *)tmp,NULL,0); } else EVP_CipherInit(evp_ctx,NULL,tmp,NULL,0);
memset(tmp,0,jj); memset(tmp,0,jj);