Add new sign and verify members to RSA_METHOD and change SSL code to use sign

and verify rather than direct encrypt/decrypt.

CHANGES

@@ -4,6 +4,17 @@
 
  Changes between 0.9.4 and 0.9.5  [xx XXX 1999]
 
+  *) Enhance RSA_METHOD structure. Now there are two extra methods, rsa_sign
+     and rsa_verify. When the RSA_FLAGS_SIGN_VER option is set these functions
+     will be called when RSA_sign() and RSA_verify() are used. This is useful
+     if rsa_pub_dec() and rsa_priv_enc() equivalents are not available.
+     For this to work properly RSA_public_decrypt() and RSA_private_encrypt()
+     should *not* be used: RSA_sign() and RSA_verify() must be used instead.
+     This necessitated the support of an extra signature type NID_md5_sha1
+     for SSL signatures and modifications to the SSL library to use it instead
+     of calling RSA_public_decrypt() and RSA_private_encrypt().
+     [Steve Henson]
+
   *) Add new -verify -CAfile and -CApath options to the crl program, these
      will lookup a CRL issuers certificate and verify the signature in a
      similar way to the verify program. Tidy up the crl program so it

crypto/rsa/rsa.h

@@ -91,6 +91,18 @@ typedef struct rsa_meth_st
 	int (*finish)(RSA *rsa);	/* called at free */
 	int flags;			/* RSA_METHOD_FLAG_* things */
 	char *app_data;			/* may be needed! */
+/* New sign and verify functions: some libraries don't allow arbitrary data
+ * to be signed/verified: this allows them to be used. Note: for this to work
+ * the RSA_public_decrypt() and RSA_private_encrypt() should *NOT* be used
+ * RSA_sign(), RSA_verify() should be used instead. Note: for backwards
+ * compatability this functionality is only enabled if the RSA_FLAG_SIGN_VER
+ * option is set in 'flags'.
+ */
+	int (*rsa_sign)(int type, unsigned char *m, unsigned int m_len,
+             unsigned char *sigret, unsigned int *siglen, RSA *rsa);
+	int (*rsa_verify)(int dtype, unsigned char *m, unsigned int m_len,
+             unsigned char *sigbuf, unsigned int siglen, RSA *rsa);
+
 	} RSA_METHOD;
 
 struct rsa_st
@@ -140,6 +152,10 @@ struct rsa_st
  */
 #define RSA_FLAG_EXT_PKEY		0x20
 
+/* This flag in the RSA_METHOD enables the new rsa_sign, rsa_verify functions.
+ */
+#define RSA_FLAG_SIGN_VER		0x40
+
 #define RSA_PKCS1_PADDING	1
 #define RSA_SSLV23_PADDING	2
 #define RSA_NO_PADDING		3
@@ -299,6 +315,7 @@ char *RSA_get_ex_data(RSA *r, int idx);
 #define RSA_R_DMP1_NOT_CONGRUENT_TO_D			 124
 #define RSA_R_DMQ1_NOT_CONGRUENT_TO_D			 125
 #define RSA_R_D_E_NOT_CONGRUENT_TO_1			 123
+#define RSA_R_INVALID_MESSAGE_LENGTH			 131
 #define RSA_R_IQMP_NOT_INVERSE_OF_Q			 126
 #define RSA_R_KEY_SIZE_TOO_SMALL			 120
 #define RSA_R_NULL_BEFORE_BLOCK_MISSING			 113

crypto/rsa/rsa_err.c

@@ -111,6 +111,7 @@ static ERR_STRING_DATA RSA_str_reasons[]=
 {RSA_R_DMP1_NOT_CONGRUENT_TO_D           ,"dmp1 not congruent to d"},
 {RSA_R_DMQ1_NOT_CONGRUENT_TO_D           ,"dmq1 not congruent to d"},
 {RSA_R_D_E_NOT_CONGRUENT_TO_1            ,"d e not congruent to 1"},
+{RSA_R_INVALID_MESSAGE_LENGTH            ,"invalid message length"},
 {RSA_R_IQMP_NOT_INVERSE_OF_Q             ,"iqmp not inverse of q"},
 {RSA_R_KEY_SIZE_TOO_SMALL                ,"key size too small"},
 {RSA_R_NULL_BEFORE_BLOCK_MISSING         ,"null before block missing"},

crypto/rsa/rsa_sign.c

@@ -63,16 +63,29 @@
 #include <openssl/objects.h>
 #include <openssl/x509.h>
 
+/* Size of an SSL signature: MD5+SHA1 */
+#define SSL_SIG_LENGTH	36
+
 int RSA_sign(int type, unsigned char *m, unsigned int m_len,
 	     unsigned char *sigret, unsigned int *siglen, RSA *rsa)
 	{
 	X509_SIG sig;
 	ASN1_TYPE parameter;
 	int i,j,ret=1;
-	unsigned char *p,*s;
+	unsigned char *p,*s = NULL;
 	X509_ALGOR algor;
 	ASN1_OCTET_STRING digest;
-
+	if(rsa->flags & RSA_FLAG_SIGN_VER)
+	      return rsa->meth->rsa_sign(type, m, m_len, sigret, siglen, rsa);
+	/* Special case: SSL signature, just check the length */
+	if(type == NID_md5_sha1) {
+		if(m_len != SSL_SIG_LENGTH) {
+			RSAerr(RSA_F_RSA_SIGN,RSA_R_INVALID_MESSAGE_LENGTH);
+			return(0);
+		}
+		i = SSL_SIG_LENGTH;
+		s = m;
+	} else {
 		sig.algor= &algor;
 		sig.algor->algorithm=OBJ_nid2obj(type);
 		if (sig.algor->algorithm == NULL)
@@ -94,12 +107,14 @@ int RSA_sign(int type, unsigned char *m, unsigned int m_len,
 		sig.digest->length=m_len;
 
 		i=i2d_X509_SIG(&sig,NULL);
+	}
 	j=RSA_size(rsa);
 	if ((i-RSA_PKCS1_PADDING) > j)
 		{
 		RSAerr(RSA_F_RSA_SIGN,RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY);
 		return(0);
 		}
+	if(type != NID_md5_sha1) {
 		s=(unsigned char *)Malloc((unsigned int)j+1);
 		if (s == NULL)
 			{
@@ -108,14 +123,17 @@ int RSA_sign(int type, unsigned char *m, unsigned int m_len,
 			}
 		p=s;
 		i2d_X509_SIG(&sig,&p);
+	}
 	i=RSA_private_encrypt(i,s,sigret,rsa,RSA_PKCS1_PADDING);
 	if (i <= 0)
 		ret=0;
 	else
 		*siglen=i;
 
+	if(type != NID_md5_sha1) {
 		memset(s,0,(unsigned int)j+1);
 		Free(s);
+	}
 	return(ret);
 	}
 
@@ -132,16 +150,29 @@ int RSA_verify(int dtype, unsigned char *m, unsigned int m_len,
 		return(0);
 		}
 
+	if(rsa->flags & RSA_FLAG_SIGN_VER)
+	    return rsa->meth->rsa_verify(dtype, m, m_len, sigbuf, siglen, rsa);
+
 	s=(unsigned char *)Malloc((unsigned int)siglen);
 	if (s == NULL)
 		{
 		RSAerr(RSA_F_RSA_VERIFY,ERR_R_MALLOC_FAILURE);
 		goto err;
 		}
+	if((dtype == NID_md5_sha1) && (m_len != SSL_SIG_LENGTH) ) {
+			RSAerr(RSA_F_RSA_VERIFY,RSA_R_INVALID_MESSAGE_LENGTH);
+			return(0);
+	}
 	i=RSA_public_decrypt((int)siglen,sigbuf,s,rsa,RSA_PKCS1_PADDING);
 
 	if (i <= 0) goto err;
 
+	/* Special case: SSL signature */
+	if(dtype == NID_md5_sha1) {
+		if((i != SSL_SIG_LENGTH) || memcmp(s, m, SSL_SIG_LENGTH))
+				RSAerr(RSA_F_RSA_VERIFY,RSA_R_BAD_SIGNATURE);
+		else ret = 1;
+	} else {
 		p=s;
 		sig=d2i_X509_SIG(NULL,&p,(long)i);
 
@@ -168,7 +199,8 @@ int RSA_verify(int dtype, unsigned char *m, unsigned int m_len,
 				}
 			else
 				{
-			RSAerr(RSA_F_RSA_VERIFY,RSA_R_ALGORITHM_MISMATCH);
+				RSAerr(RSA_F_RSA_VERIFY,
+						RSA_R_ALGORITHM_MISMATCH);
 				goto err;
 				}
 			}
@@ -179,6 +211,7 @@ int RSA_verify(int dtype, unsigned char *m, unsigned int m_len,
 			}
 		else
 			ret=1;
+	}
 err:
 	if (sig != NULL) X509_SIG_free(sig);
 	memset(s,0,(unsigned int)siglen);

ssl/s3_clnt.c

@@ -1053,15 +1053,15 @@ static int ssl3_get_key_exchange(SSL *s)
 				q+=i;
 				j+=i;
 				}
-			i=RSA_public_decrypt((int)n,p,p,pkey->pkey.rsa,
+			i=RSA_verify(NID_md5_sha1, md_buf, j, p, n,
-				RSA_PKCS1_PADDING);
+								pkey->pkey.rsa);
-			if (i <= 0)
+			if (i < 0)
 				{
 				al=SSL_AD_DECRYPT_ERROR;
 				SSLerr(SSL_F_SSL3_GET_KEY_EXCHANGE,SSL_R_BAD_RSA_DECRYPT);
 				goto f_err;
 				}
-			if ((j != i) || (memcmp(p,md_buf,i) != 0))
+			if (i == 0)
 				{
 				/* bad signature */
 				al=SSL_AD_DECRYPT_ERROR;
@@ -1481,11 +1481,9 @@ static int ssl3_send_client_verify(SSL *s)
 			{
 			s->method->ssl3_enc->cert_verify_mac(s,
 				&(s->s3->finish_dgst1),&(data[0]));
-			i=RSA_private_encrypt(
+			if (RSA_sign(NID_md5_sha1, data,
 					 MD5_DIGEST_LENGTH+SHA_DIGEST_LENGTH,
-				data,&(p[2]),pkey->pkey.rsa,
+					&(p[2]), &i, pkey->pkey.rsa) <= 0 )
-				RSA_PKCS1_PADDING);
-			if (i <= 0)
 				{
 				SSLerr(SSL_F_SSL3_SEND_CLIENT_VERIFY,ERR_R_RSA_LIB);
 				goto err;

ssl/s3_srvr.c

@@ -1026,9 +1026,8 @@ static int ssl3_send_server_key_exchange(SSL *s)
 					q+=i;
 					j+=i;
 					}
-				i=RSA_private_encrypt(j,md_buf,&(p[2]),
+				if (RSA_sign(NID_md5_sha1, md_buf, j,
-					pkey->pkey.rsa,RSA_PKCS1_PADDING);
+					&(p[2]), &i, pkey->pkey.rsa) <= 0)
-				if (i <= 0)
 					{
 					SSLerr(SSL_F_SSL3_SEND_SERVER_KEY_EXCHANGE,ERR_LIB_RSA);
 					goto err;
@@ -1449,16 +1448,16 @@ static int ssl3_get_cert_verify(SSL *s)
 #ifndef NO_RSA 
 	if (pkey->type == EVP_PKEY_RSA)
 		{
-		i=RSA_public_decrypt(i,p,p,pkey->pkey.rsa,RSA_PKCS1_PADDING);
+		i=RSA_verify(NID_md5_sha1, s->s3->tmp.finish_md,
+			MD5_DIGEST_LENGTH+SHA_DIGEST_LENGTH, p, i, 
+							pkey->pkey.rsa);
 		if (i < 0)
 			{
 			al=SSL_AD_DECRYPT_ERROR;
 			SSLerr(SSL_F_SSL3_GET_CERT_VERIFY,SSL_R_BAD_RSA_DECRYPT);
 			goto f_err;
 			}
-		if ((i != (MD5_DIGEST_LENGTH+SHA_DIGEST_LENGTH)) ||
+		if (i == 0)
-			memcmp(&(s->s3->tmp.finish_md[0]),p,
-				MD5_DIGEST_LENGTH+SHA_DIGEST_LENGTH))
 			{
 			al=SSL_AD_DECRYPT_ERROR;
 			SSLerr(SSL_F_SSL3_GET_CERT_VERIFY,SSL_R_BAD_RSA_SIGNATURE);