Fix various less obvious bugs in PKCS#7 handling: such as not zeroing

the secret key before we've encrypted it and using the right NID for RC2-64.
Add various arguments to the experimental programs 'dec' and 'enc' to make
testing less painful.

This stuff has now been tested against Netscape Messenger and it can encrypt
and decrypt S/MIME messages with RC2 (128, 64 and 40 bit) DES and triple DES.

Its still experimental though...

CHANGES

@@ -10,6 +10,10 @@
                                    [23-Dec-1998] down below; but in later
                                    versions, these hyphens are gone.]
 
+  *) Fix most of the other PKCS#7 bugs. The "experimental" code can now
+     correctly handle encrypted S/MIME data.
+     [Steve Henson]
+
   *) Change type of various DES function arguments from des_cblock
      (which means, in function argument declarations, pointer to char)
      to des_cblock * (meaning pointer to array with 8 char elements),

apps/pkcs12.c

@@ -371,8 +371,6 @@ if (export_cert) {
 
 	if (canames) sk_free(canames);
 
-	/* if (!pmatch) ...?  What should happen here?  XXX */
-
 	if(!noprompt &&
 		EVP_read_pw_string(pass, 50, "Enter Export Password:", 1)) {
 	    BIO_printf (bio_err, "Can't read Password\n");
@@ -400,7 +398,7 @@ if (export_cert) {
 			cpass, -1, NULL, 0, iter, p8);
 	PKCS8_PRIV_KEY_INFO_free(p8);
         if (name) PKCS12_add_friendlyname (bag, name, -1);
-	PKCS12_add_localkeyid (bag, keyid, keyidlen);
+	if(!pmatch) PKCS12_add_localkeyid (bag, keyid, keyidlen);
 	bags = sk_new(NULL);
 	sk_push (bags, (char *)bag);
 	/* Turn it into unencrypted safe bag */

crypto/evp/e_cbc_r2.c

@@ -91,7 +91,7 @@ static EVP_CIPHER r2_cbc_cipher=
 
 static EVP_CIPHER r2_64_cbc_cipher=
 	{
-	NID_rc2_40_cbc,
+	NID_rc2_64_cbc,
 	8,8 /* 64 bit */,8,
 	rc2_cbc_init_key,
 	rc2_cbc_cipher,

crypto/pkcs7/dec.c

@@ -56,48 +56,39 @@
  * [including the GNU Public Licence.]
  */
 #include <stdio.h>
-#include <openssl/asn1.h>
+#include <stdlib.h>
 #include <openssl/bio.h>
 #include <openssl/x509.h>
 #include <openssl/pem.h>
+#include <openssl/err.h>
+#include <openssl/asn1.h>
 
 int verify_callback(int ok, X509_STORE_CTX *ctx);
 
 BIO *bio_err=NULL;
 
-main(argc,argv)
+int main(argc,argv)
 int argc;
 char *argv[];
 	{
+	char *keyfile;
 	BIO *in;
-	X509 *x509,*x;
 	EVP_PKEY *pkey;
+	X509 *x509;
 	PKCS7 *p7;
-	PKCS7_SIGNED *s;
 	PKCS7_SIGNER_INFO *si;
-	PKCS7_ISSUER_AND_SERIAL *ias;
 	X509_STORE_CTX cert_ctx;
 	X509_STORE *cert_store=NULL;
-	X509_LOOKUP *lookup=NULL;
 	BIO *data,*detached=NULL,*p7bio=NULL;
 	char buf[1024*4];
-	unsigned char *p,*pp;
+	unsigned char *pp;
-	int i,j,printit=0;
+	int i,printit=0;
 	STACK *sk;
 
 	SSLeay_add_all_algorithms();
 	bio_err=BIO_new_fp(stderr,BIO_NOCLOSE);
-	EVP_add_digest(EVP_sha1());
-	EVP_add_cipher(EVP_des_ede3_cbc());
-
-        if ((in=BIO_new_file("server.pem","r")) == NULL) goto err;
-        if ((x509=PEM_read_bio_X509(in,NULL,NULL)) == NULL) goto err;
-        BIO_reset(in);
-        if ((pkey=PEM_read_bio_PrivateKey(in,NULL,NULL)) == NULL) goto err;
-        BIO_free(in);
 
 	data=BIO_new(BIO_s_file());
-again:
 	pp=NULL;
 	while (argc > 1)
 		{
@@ -107,21 +98,33 @@ again:
 			{
 			printit=1;
 			}
-		else if ((strcmp(argv[0],"-d") == 0) && (argc >= 2))
+		else if ((strcmp(argv[0],"-k") == 0) && (argc >= 2)) {
+			keyfile = argv[1];
+			argc-=1;
+			argv+=1;
+		} else if ((strcmp(argv[0],"-d") == 0) && (argc >= 2))
 			{
 			detached=BIO_new(BIO_s_file());
 			if (!BIO_read_filename(detached,argv[1]))
 				goto err;
-			argc--;
+			argc-=1;
-			argv++;
+			argv+=1;
 			}
-		else
+		else break;
-			{
+		}
-			pp=argv[0];
+
-			if (!BIO_read_filename(data,argv[0]))
+	 if (!BIO_read_filename(data,argv[0])) goto err; 
+
+	if(!keyfile) {
+		fprintf(stderr, "No private key file specified\n");
 		goto err;
 	}
-		}
+
+        if ((in=BIO_new_file(keyfile,"r")) == NULL) goto err;
+        if ((x509=PEM_read_bio_X509(in,NULL,NULL)) == NULL) goto err;
+        BIO_reset(in);
+        if ((pkey=PEM_read_bio_PrivateKey(in,NULL,NULL)) == NULL) goto err;
+        BIO_free(in);
 
 	if (pp == NULL)
 		BIO_set_fp(data,stdin,BIO_NOCLOSE);
@@ -158,14 +161,14 @@ again:
 		i=BIO_read(p7bio,buf,sizeof(buf));
 		/* print it? */
 		if (i <= 0) break;
-		write(fileno(stdout),buf,i);
+		fwrite(buf,1, i, stdout);
 		}
 
 	/* We can now verify signatures */
 	sk=PKCS7_get_signer_info(p7);
 	if (sk == NULL)
 		{
-		printf("there are no signatures on this data\n");
+		fprintf(stderr, "there are no signatures on this data\n");
 		}
 	else
 		{

crypto/pkcs7/enc.c

@@ -59,57 +59,69 @@
 #include <openssl/bio.h>
 #include <openssl/x509.h>
 #include <openssl/pem.h>
+#include <openssl/err.h>
 
-main(argc,argv)
+int main(argc,argv)
 int argc;
 char *argv[];
 	{
 	X509 *x509;
-	EVP_PKEY *pkey;
 	PKCS7 *p7;
-	PKCS7 *p7_data;
-	PKCS7_SIGNER_INFO *si;
 	BIO *in;
 	BIO *data,*p7bio;
 	char buf[1024*4];
-	int i,j;
+	int i;
 	int nodetach=1;
+	char *keyfile = NULL;
+	const EVP_CIPHER *cipher;
 
-	EVP_add_digest(EVP_sha1());
+	SSLeay_add_all_algorithms();
-	EVP_add_cipher(EVP_des_ede3_cbc());
 
 	data=BIO_new(BIO_s_file());
-again:
+	while(argc > 1)
-	if (argc > 1)
 		{
 		if (strcmp(argv[1],"-nd") == 0)
 			{
 			nodetach=1;
 			argv++; argc--;
-			goto again;
 			}
-		if (!BIO_read_filename(data,argv[1]))
+		else if ((strcmp(argv[1],"-c") == 0) && (argc >= 2)) {
+			if(!(cipher = EVP_get_cipherbyname(argv[2]))) {
+				fprintf(stderr, "Unknown cipher %s\n", argv[2]);
 				goto err;
 			}
-	else
+			argc-=2;
-		BIO_set_fp(data,stdin,BIO_NOCLOSE);
+			argv+=2;
+		} else if ((strcmp(argv[1],"-k") == 0) && (argc >= 2)) {
+			keyfile = argv[2];
+			argc-=2;
+			argv+=2;
+		} else break;
+	}
 
-	if ((in=BIO_new_file("server.pem","r")) == NULL) goto err;
+	if (!BIO_read_filename(data,argv[1])) goto err;
+
+	if ((in=BIO_new_file(keyfile,"r")) == NULL) goto err;
 	if ((x509=PEM_read_bio_X509(in,NULL,NULL)) == NULL) goto err;
+
+	p7=PKCS7_new();
+#if 0
 	BIO_reset(in);
 	if ((pkey=PEM_read_bio_PrivateKey(in,NULL,NULL)) == NULL) goto err;
 	BIO_free(in);
-
-	p7=PKCS7_new();
 	PKCS7_set_type(p7,NID_pkcs7_signedAndEnveloped);
 	 
 	if (PKCS7_add_signature(p7,x509,pkey,EVP_sha1()) == NULL) goto err;
-
-	if (!PKCS7_set_cipher(p7,EVP_des_ede3_cbc())) goto err;
-	if (PKCS7_add_recipient(p7,x509) == NULL) goto err;
-
 	/* we may want to add more */
 	PKCS7_add_certificate(p7,x509);
+#else
+	PKCS7_set_type(p7,NID_pkcs7_enveloped);
+#endif
+	if(!cipher) cipher = EVP_des_ede3_cbc();
+
+	if (!PKCS7_set_cipher(p7,cipher)) goto err;
+	if (PKCS7_add_recipient(p7,x509) == NULL) goto err;
+
 
 
 	/* Set the content of the signed to 'data' */

crypto/pkcs7/pk7_doit.c

@@ -159,12 +159,11 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
 		keylen=EVP_CIPHER_key_length(evp_cipher);
 		ivlen=EVP_CIPHER_iv_length(evp_cipher);
 		RAND_bytes(key,keylen);
-		EVP_CipherInit(ctx, evp_cipher, key, iv, 1);
-		memset(key, 0, keylen);
 		xalg->algorithm = OBJ_nid2obj(EVP_CIPHER_type(evp_cipher));
+		if (ivlen > 0) RAND_bytes(iv,ivlen);
+		EVP_CipherInit(ctx, evp_cipher, key, iv, 1);
 
 		if (ivlen > 0) {
-			RAND_bytes(iv,ivlen);
 			if (xalg->parameter == NULL) 
 						xalg->parameter=ASN1_TYPE_new();
 			if(EVP_CIPHER_param_to_asn1(ctx, xalg->parameter) < 0)
@@ -206,6 +205,7 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
 			ASN1_OCTET_STRING_set(ri->enc_key,tmp,jj);
 			}
 		Free(tmp);
+		memset(key, 0, keylen);
 
 		if (out == NULL)
 			out=btmp;