Reorganise supported signature algorithm extension processing.

Only store encoded versions of peer and configured signature algorithms.
Determine shared signature algorithms and cache the result along with NID
equivalents of each algorithm.
(backport from HEAD)

ssl/ssl.h

@@ -2532,6 +2532,7 @@ void ERR_load_SSL_strings(void);
 #define SSL_R_NO_RENEGOTIATION				 339
 #define SSL_R_NO_REQUIRED_DIGEST			 324
 #define SSL_R_NO_SHARED_CIPHER				 193
+#define SSL_R_NO_SHARED_SIGATURE_ALGORITHMS		 376
 #define SSL_R_NO_SRTP_PROFILES				 359
 #define SSL_R_NO_VERIFY_CALLBACK			 194
 #define SSL_R_NULL_SSL_CTX				 195

ssl/ssl_cert.c

@@ -160,7 +160,7 @@ int SSL_get_ex_data_X509_STORE_CTX_idx(void)
 	return ssl_x509_store_ctx_idx;
 	}
 
-static void ssl_cert_set_default_md(CERT *cert)
+void ssl_cert_set_default_md(CERT *cert)
 	{
 	/* Set digest values to defaults */
 #ifndef OPENSSL_NO_DSA
@@ -373,6 +373,8 @@ CERT *ssl_cert_dup(CERT *cert)
 		}
 	else
 		ret->conf_sigalgs = NULL;
+	/* Shared sigalgs also NULL */
+	ret->shared_sigalgs = NULL;
 
 	return(ret);
 	
@@ -464,6 +466,8 @@ void ssl_cert_free(CERT *c)
 		OPENSSL_free(c->peer_sigalgs);
 	if (c->conf_sigalgs)
 		OPENSSL_free(c->conf_sigalgs);
+	if (c->shared_sigalgs)
+		OPENSSL_free(c->shared_sigalgs);
 	OPENSSL_free(c);
 	}
 

ssl/ssl_err.c

@@ -463,6 +463,7 @@ static ERR_STRING_DATA SSL_str_reasons[]=
 {ERR_REASON(SSL_R_NO_RENEGOTIATION)      ,"no renegotiation"},
 {ERR_REASON(SSL_R_NO_REQUIRED_DIGEST)    ,"digest requred for handshake isn't computed"},
 {ERR_REASON(SSL_R_NO_SHARED_CIPHER)      ,"no shared cipher"},
+{ERR_REASON(SSL_R_NO_SHARED_SIGATURE_ALGORITHMS),"no shared sigature algorithms"},
 {ERR_REASON(SSL_R_NO_SRTP_PROFILES)      ,"no srtp profiles"},
 {ERR_REASON(SSL_R_NO_VERIFY_CALLBACK)    ,"no verify callback"},
 {ERR_REASON(SSL_R_NULL_SSL_CTX)          ,"null ssl ctx"},

ssl/ssl_locl.h

@@ -523,15 +523,20 @@ typedef struct cert_st
 	 * algorithms extension for server or as part of a certificate
 	 * request for client.
 	 */
-	TLS_SIGALGS *peer_sigalgs;
+	unsigned char *peer_sigalgs;
 	/* Size of above array */
 	size_t peer_sigalgslen;
 	/* configured signature algorithms (can be NULL for default).
 	 * sent in signature algorithms extension or certificate request.
 	 */
-	TLS_SIGALGS *conf_sigalgs;
+	unsigned char *conf_sigalgs;
 	/* Size of above array */
 	size_t conf_sigalgslen;
+	/* Signature algorithms shared by client and server: cached
+	 * because these are used most often
+	 */
+	TLS_SIGALGS *shared_sigalgs;
+	size_t shared_sigalgslen;
 
 	int references; /* >1 only if SSL_copy_session_id is used */
 	} CERT;
@@ -841,6 +846,7 @@ void ssl_clear_cipher_ctx(SSL *s);
 int ssl_clear_bad_session(SSL *s);
 CERT *ssl_cert_new(void);
 CERT *ssl_cert_dup(CERT *cert);
+void ssl_cert_set_default_md(CERT *cert);
 int ssl_cert_inst(CERT **o);
 void ssl_cert_clear_certs(CERT *c);
 void ssl_cert_free(CERT *c);

ssl/t1_lib.c

@@ -631,35 +631,26 @@ static unsigned char tls12_sigalgs[] = {
 
 size_t tls12_get_sig_algs(SSL *s, unsigned char *p)
 	{
-	TLS_SIGALGS *sptr = s->cert->conf_sigalgs;
-	size_t slen;
+	const unsigned char *sigs;
+	size_t sigslen;
+	sigs = s->cert->conf_sigalgs;
 
-	/* Use custom signature algorithms if any are set */
-
-	if (sptr)
+	if (sigs)
+		sigslen = s->cert->conf_sigalgslen;
+	else
 		{
-		slen = s->cert->conf_sigalgslen;
-		if (p)
-			{
-			size_t i;
-			for (i = 0; i < slen; i++, sptr++)
-				{
-				*p++ = sptr->rhash;
-				*p++ = sptr->rsign;
-				}
-			}
-		return slen * 2;
-		}
-		
-	slen = sizeof(tls12_sigalgs);
+		sigs = tls12_sigalgs;
+		sigslen = sizeof(tls12_sigalgs);
 #ifdef OPENSSL_FIPS
-	/* If FIPS mode don't include MD5 which is last */
-	if (FIPS_mode())
-		slen -= 2;
+		/* If FIPS mode don't include MD5 which is last */
+		if (FIPS_mode())
+			sigslen -= 2;
 #endif
+		}
+
 	if (p)
-		memcpy(p, tls12_sigalgs, slen);
-	return slen;
+		memcpy(p, sigs, sigslen);
+	return sigslen;
 	}
 
 /* byte_compare is a compare function for qsort(3) that compares bytes. */
@@ -1312,7 +1303,6 @@ static int ssl_scan_clienthello_tlsext(SSL *s, unsigned char **p, unsigned char
 	unsigned short len;
 	unsigned char *data = *p;
 	int renegotiate_seen = 0;
-	int sigalg_seen = 0;
 
 	s->servername_done = 0;
 	s->tlsext_status_type = -1;
@@ -1324,6 +1314,18 @@ static int ssl_scan_clienthello_tlsext(SSL *s, unsigned char **p, unsigned char
 	s->tlsext_heartbeat &= ~(SSL_TLSEXT_HB_ENABLED |
 	                       SSL_TLSEXT_HB_DONT_SEND_REQUESTS);
 #endif
+	/* Clear any signature algorithms extension received */
+	if (s->cert->peer_sigalgs)
+		{
+		OPENSSL_free(s->cert->peer_sigalgs);
+		s->cert->peer_sigalgs = NULL;
+		}
+	/* Clear any shared sigtnature algorithms */
+	if (s->cert->shared_sigalgs)
+		{
+		OPENSSL_free(s->cert->shared_sigalgs);
+		s->cert->shared_sigalgs = NULL;
+		}
 
 	if (data >= (d+n-2))
 		goto ri_check;
@@ -1600,15 +1602,14 @@ static int ssl_scan_clienthello_tlsext(SSL *s, unsigned char **p, unsigned char
 		else if (type == TLSEXT_TYPE_signature_algorithms)
 			{
 			int dsize;
-			if (sigalg_seen || size < 2) 
+			if (s->cert->peer_sigalgs || size < 2) 
 				{
 				*al = SSL_AD_DECODE_ERROR;
 				return 0;
 				}
-			sigalg_seen = 1;
 			n2s(data,dsize);
 			size -= 2;
-			if (dsize != size || dsize & 1) 
+			if (dsize != size || dsize & 1 || !dsize) 
 				{
 				*al = SSL_AD_DECODE_ERROR;
 				return 0;
@@ -1618,6 +1619,16 @@ static int ssl_scan_clienthello_tlsext(SSL *s, unsigned char **p, unsigned char
 				*al = SSL_AD_DECODE_ERROR;
 				return 0;
 				}
+			/* If sigalgs received and no shared algorithms fatal
+			 * error.
+			 */
+			if (s->cert->peer_sigalgs && !s->cert->shared_sigalgs)
+				{
+				SSLerr(SSL_F_SSL_SCAN_CLIENTHELLO_TLSEXT,
+					SSL_R_NO_SHARED_SIGATURE_ALGORITHMS);
+				*al = SSL_AD_ILLEGAL_PARAMETER;
+				return 0;
+				}
 			}
 		else if (type == TLSEXT_TYPE_status_request &&
 		         s->version != DTLS1_VERSION && s->ctx->tlsext_status_cb)
@@ -1856,6 +1867,9 @@ static int ssl_scan_clienthello_tlsext(SSL *s, unsigned char **p, unsigned char
 				SSL_R_UNSAFE_LEGACY_RENEGOTIATION_DISABLED);
 		return 0;
 		}
+	/* If no signature algorithms extension set default values */
+	if (!s->cert->peer_sigalgs)
+		ssl_cert_set_default_md(s->cert);
 
 	return 1;
 	}
@@ -2898,11 +2912,145 @@ const EVP_MD *tls12_get_hash(unsigned char hash_alg)
 		}
 	}
 
+static int tls12_get_pkey_idx(unsigned char sig_alg)
+	{
+	switch(sig_alg)
+		{
+#ifndef OPENSSL_NO_RSA
+	case TLSEXT_signature_rsa:
+		return SSL_PKEY_RSA_SIGN;
+#endif
+#ifndef OPENSSL_NO_DSA
+	case TLSEXT_signature_dsa:
+		return SSL_PKEY_DSA_SIGN;
+#endif
+#ifndef OPENSSL_NO_ECDSA
+	case TLSEXT_signature_ecdsa:
+		return SSL_PKEY_ECC;
+#endif
+		}
+	return -1;
+	}
+
+/* Convert TLS 1.2 signature algorithm extension values into NIDs */
+static void tls1_lookup_sigalg(int *phash_nid, int *psign_nid,
+			int *psignhash_nid, const unsigned char *data)
+	{
+	int sign_nid, hash_nid;
+	if (!phash_nid && !psign_nid && !psignhash_nid)
+		return;
+	if (phash_nid || psignhash_nid)
+		{
+		hash_nid = tls12_find_nid(data[0], tls12_md,
+					sizeof(tls12_md)/sizeof(tls12_lookup));
+		if (phash_nid)
+			*phash_nid = hash_nid;
+		}
+	if (psign_nid || psignhash_nid)
+		{
+		sign_nid = tls12_find_nid(data[1], tls12_sig,
+					sizeof(tls12_sig)/sizeof(tls12_lookup));
+		if (psign_nid)
+			*psign_nid = sign_nid;
+		}
+	if (psignhash_nid)
+		{
+		if (sign_nid && hash_nid)
+			OBJ_find_sigid_by_algs(psignhash_nid,
+							hash_nid, sign_nid);
+		else
+			*psignhash_nid = NID_undef;
+		}
+	}
+/* Given preference and allowed sigalgs set shared sigalgs */
+static int tls12_do_shared_sigalgs(TLS_SIGALGS *shsig,
+				const unsigned char *pref, size_t preflen,
+				const unsigned char *allow, size_t allowlen)
+	{
+	const unsigned char *ptmp, *atmp;
+	size_t i, j, nmatch = 0;
+	for (i = 0, ptmp = pref; i < preflen; i+=2, ptmp+=2)
+		{
+		/* Skip disabled hashes or signature algorithms */
+		if (tls12_get_hash(ptmp[0]) == NULL)
+			continue;
+		if (tls12_get_pkey_idx(ptmp[1]) == -1)
+			continue;
+		for (j = 0, atmp = allow; j < allowlen; j+=2, atmp+=2)
+			{
+			if (ptmp[0] == atmp[0] && ptmp[1] == atmp[1])
+				{
+				nmatch++;
+				if (shsig)
+					{
+					shsig->rhash = ptmp[0];
+					shsig->rsign = ptmp[1];
+					tls1_lookup_sigalg(&shsig->hash_nid,
+						&shsig->sign_nid,
+						&shsig->signandhash_nid,
+						ptmp);
+					shsig++;
+					}
+				break;
+				}
+			}
+		}
+	return nmatch;
+	}
+
+/* Set shared signature algorithms for SSL structures */
+static int tls1_set_shared_sigalgs(SSL *s)
+	{
+	const unsigned char *pref, *allow, *conf;
+	size_t preflen, allowlen, conflen;
+	size_t nmatch;
+	TLS_SIGALGS *salgs = NULL;
+	CERT *c = s->cert;
+	conf = c->conf_sigalgs;
+	if (conf)
+		conflen = c->conf_sigalgslen;
+	else
+		{
+		conf = tls12_sigalgs;
+		conflen = sizeof(tls12_sigalgs);
+#ifdef OPENSSL_FIPS
+		if (FIPS_mode())
+			conflen -= 2;
+#endif
+		}
+	if(s->options & SSL_OP_CIPHER_SERVER_PREFERENCE)
+		{
+		pref = conf;
+		preflen = conflen;
+		allow = c->peer_sigalgs;
+		allowlen = c->peer_sigalgslen;
+		}
+	else
+		{
+		allow = conf;
+		allowlen = conflen;
+		pref = c->peer_sigalgs;
+		preflen = c->peer_sigalgslen;
+		}
+	nmatch = tls12_do_shared_sigalgs(NULL, pref, preflen, allow, allowlen);
+	if (!nmatch)
+		return 1;
+	salgs = OPENSSL_malloc(nmatch * sizeof(TLS_SIGALGS));
+	if (!salgs)
+		return 0;
+	nmatch = tls12_do_shared_sigalgs(salgs, pref, preflen, allow, allowlen);
+	c->shared_sigalgs = salgs;
+	c->shared_sigalgslen = nmatch;
+	return 1;
+	}
+		
+
 /* Set preferred digest for each key type */
 
 int tls1_process_sigalgs(SSL *s, const unsigned char *data, int dsize)
 	{
-	int i, idx;
+	int idx;
+	size_t i;
 	const EVP_MD *md;
 	CERT *c = s->cert;
 	TLS_SIGALGS *sigptr;
@@ -2918,60 +3066,27 @@ int tls1_process_sigalgs(SSL *s, const unsigned char *data, int dsize)
 	c->pkeys[SSL_PKEY_RSA_ENC].digest = NULL;
 	c->pkeys[SSL_PKEY_ECC].digest = NULL;
 
-	if (c->peer_sigalgs)
-		OPENSSL_free(c->peer_sigalgs);
-	c->peer_sigalgs = OPENSSL_malloc((dsize/2) * sizeof(TLS_SIGALGS));
+	c->peer_sigalgs = OPENSSL_malloc(dsize);
 	if (!c->peer_sigalgs)
 		return 0;
-	c->peer_sigalgslen = dsize/2;
+	c->peer_sigalgslen = dsize;
+	memcpy(c->peer_sigalgs, data, dsize);
 
-	for (i = 0, sigptr = c->peer_sigalgs; i < dsize; i += 2, sigptr++)
+	tls1_set_shared_sigalgs(s);
+
+	for (i = 0, sigptr = c->shared_sigalgs;
+			i < c->shared_sigalgslen; i++, sigptr++)
 		{
-		sigptr->rhash = data[i];
-		sigptr->rsign = data[i + 1];
-		sigptr->hash_nid = tls12_find_nid(sigptr->rhash, tls12_md,
-					sizeof(tls12_md)/sizeof(tls12_lookup));
-		sigptr->sign_nid = tls12_find_nid(sigptr->rsign, tls12_sig,
-					sizeof(tls12_sig)/sizeof(tls12_lookup));
-		if (!OBJ_find_sigid_by_algs(&sigptr->signandhash_nid,
-						sigptr->hash_nid,
-						sigptr->sign_nid))
-			sigptr->signandhash_nid = NID_undef;
-		switch(sigptr->rsign)
-			{
-#ifndef OPENSSL_NO_RSA
-			case TLSEXT_signature_rsa:
-			idx = SSL_PKEY_RSA_SIGN;
-			break;
-#endif
-#ifndef OPENSSL_NO_DSA
-			case TLSEXT_signature_dsa:
-			idx = SSL_PKEY_DSA_SIGN;
-			break;
-#endif
-#ifndef OPENSSL_NO_ECDSA
-			case TLSEXT_signature_ecdsa:
-			idx = SSL_PKEY_ECC;
-			break;
-#endif
-			default:
-			continue;
-			}
-
-		if (c->pkeys[idx].digest == NULL)
+		idx = tls12_get_pkey_idx(sigptr->rsign);
+		if (idx > 0 && c->pkeys[idx].digest == NULL)
 			{
 			md = tls12_get_hash(sigptr->rhash);
-			if (md)
-				{
-				c->pkeys[idx].digest = md;
-				if (idx == SSL_PKEY_RSA_SIGN)
-					c->pkeys[SSL_PKEY_RSA_ENC].digest = md;
-				}
+			c->pkeys[idx].digest = md;
+			if (idx == SSL_PKEY_RSA_SIGN)
+				c->pkeys[SSL_PKEY_RSA_ENC].digest = md;
 			}
 
 		}
-
-
 	/* Set any remaining keys to default values. NOTE: if alg is not
 	 * supported it stays as NULL.
 	 */
@@ -2993,32 +3108,48 @@ int tls1_process_sigalgs(SSL *s, const unsigned char *data, int dsize)
 	return 1;
 	}
 
-#endif
 
 int SSL_get_sigalgs(SSL *s, int idx,
-			int *psign, int *phash, int *psignandhash,
+			int *psign, int *phash, int *psignhash,
 			unsigned char *rsig, unsigned char *rhash)
 	{
-	if (s->cert->peer_sigalgs == NULL)
+	const unsigned char *psig = s->cert->peer_sigalgs;
+	if (psig == NULL)
 		return 0;
 	if (idx >= 0)
 		{
-		TLS_SIGALGS *psig;
+		idx <<= 1;
 		if (idx >= (int)s->cert->peer_sigalgslen)
 			return 0;
-		psig = s->cert->peer_sigalgs + idx;
-		if (psign)
-			*psign = psig->sign_nid;
-		if (phash)
-			*phash = psig->hash_nid;
-		if (psignandhash)
-			*psignandhash = psig->signandhash_nid;
-		if (rsig)
-			*rsig = psig->rsign;
+		psig += idx;
 		if (rhash)
-			*rhash = psig->rhash;
+			*rhash = psig[0];
+		if (rsig)
+			*rsig = psig[1];
+		tls1_lookup_sigalg(phash, psign, psignhash, psig);
 		}
-	return s->cert->peer_sigalgslen;
+	return s->cert->peer_sigalgslen / 2;
+	}
+
+int SSL_get_shared_sigalgs(SSL *s, int idx,
+			int *psign, int *phash, int *psignhash,
+			unsigned char *rsig, unsigned char *rhash)
+	{
+	TLS_SIGALGS *shsigalgs = s->cert->shared_sigalgs;
+	if (!shsigalgs || idx >= (int)s->cert->shared_sigalgslen)
+		return 0;
+	shsigalgs += idx;
+	if (phash)
+		*phash = shsigalgs->hash_nid;
+	if (psign)
+		*psign = shsigalgs->sign_nid;
+	if (psignhash)
+		*psignhash = shsigalgs->signandhash_nid;
+	if (rsig)
+		*rsig = shsigalgs->rsign;
+	if (rhash)
+		*rhash = shsigalgs->rhash;
+	return s->cert->shared_sigalgslen;
 	}
 	
 
@@ -3167,7 +3298,7 @@ tls1_heartbeat(SSL *s)
 	}
 #endif
 
-#define MAX_SIGALGLEN	(TLSEXT_hash_num * TLSEXT_signature_num *2)
+#define MAX_SIGALGLEN	(TLSEXT_hash_num * TLSEXT_signature_num * 2)
 
 typedef struct
 	{
@@ -3231,35 +3362,27 @@ int tls1_set_sigalgs_list(CERT *c, const char *str)
 	return tls1_set_sigalgs(c, sig.sigalgs, sig.sigalgcnt);
 	}
 
-int tls1_set_sigalgs(CERT *c, const int *salg, size_t salglen)
+int tls1_set_sigalgs(CERT *c, const int *psig_nids, size_t salglen)
 	{
-	TLS_SIGALGS *sigalgs, *sptr;
+	unsigned char *sigalgs, *sptr;
 	int rhash, rsign;
 	size_t i;
 	if (salglen & 1)
 		return 0;
-	salglen /= 2;
-	sigalgs = OPENSSL_malloc(sizeof(TLS_SIGALGS) * salglen);
+	sigalgs = OPENSSL_malloc(salglen);
 	if (sigalgs == NULL)
 		return 0;
-	for (i = 0, sptr = sigalgs; i < salglen; i++, sptr++)
+	for (i = 0, sptr = sigalgs; i < salglen; i+=2)
 		{
-		sptr->hash_nid = *salg++;
-		sptr->sign_nid = *salg++;
-		rhash = tls12_find_id(sptr->hash_nid, tls12_md,
+		rhash = tls12_find_id(*psig_nids++, tls12_md,
 					sizeof(tls12_md)/sizeof(tls12_lookup));
-		rsign = tls12_find_id(sptr->sign_nid, tls12_sig,
+		rsign = tls12_find_id(*psig_nids++, tls12_sig,
 				sizeof(tls12_sig)/sizeof(tls12_lookup));
 
 		if (rhash == -1 || rsign == -1)
 			goto err;
-
-		if (!OBJ_find_sigid_by_algs(&sptr->signandhash_nid,
-						sptr->hash_nid,
-						sptr->sign_nid))
-			sptr->signandhash_nid = NID_undef;
-		sptr->rhash = rhash;
-		sptr->rsign = rsign;
+		*sptr++ = rhash;
+		*sptr++ = rsign;
 		}
 
 	if (c->conf_sigalgs)
@@ -3273,3 +3396,5 @@ int tls1_set_sigalgs(CERT *c, const int *salg, size_t salglen)
 	OPENSSL_free(sigalgs);
 	return 0;
 	}
+
+#endif

ssl/tls1.h

@@ -314,6 +314,10 @@ int SSL_get_sigalgs(SSL *s, int idx,
 			int *psign, int *phash, int *psignandhash,
 			unsigned char *rsig, unsigned char *rhash);
 
+int SSL_get_shared_sigalgs(SSL *s, int idx,
+			int *psign, int *phash, int *psignandhash,
+			unsigned char *rsig, unsigned char *rhash);
+
 #define SSL_set_tlsext_host_name(s,name) \
 SSL_ctrl(s,SSL_CTRL_SET_TLSEXT_HOSTNAME,TLSEXT_NAMETYPE_host_name,(char *)name)