Curl_ssl_md5sum: return CURLcode

... since the funciton can fail on OOM. Check this return code.

Coverity CID 1291705.

lib/curl_ntlm_msgs.c

@@ -591,10 +591,10 @@ CURLcode Curl_sasl_create_ntlm_type3_message(struct SessionHandle *data,
     memcpy(tmp, &ntlm->nonce[0], 8);
     memcpy(tmp + 8, entropy, 8);
 
-    Curl_ssl_md5sum(tmp, 16, md5sum, MD5_DIGEST_LENGTH);
-
-    /* We shall only use the first 8 bytes of md5sum, but the des
-       code in Curl_ntlm_core_lm_resp only encrypt the first 8 bytes */
+    result = Curl_ssl_md5sum(tmp, 16, md5sum, MD5_DIGEST_LENGTH);
+    if(!result)
+      /* We shall only use the first 8 bytes of md5sum, but the des code in
+         Curl_ntlm_core_lm_resp only encrypt the first 8 bytes */
       result = Curl_ntlm_core_mk_nt_hash(data, passwdp, ntbuffer);
     if(result)
       return result;

lib/vtls/vtls.c

@@ -828,7 +828,7 @@ CURLcode Curl_pin_peer_pubkey(const char *pinnedpubkey,
 }
 
 #ifndef CURL_DISABLE_CRYPTO_AUTH
-void Curl_ssl_md5sum(unsigned char *tmp, /* input */
+CURLcode Curl_ssl_md5sum(unsigned char *tmp, /* input */
                          size_t tmplen,
                          unsigned char *md5sum, /* output */
                          size_t md5len)
@@ -841,9 +841,12 @@ void Curl_ssl_md5sum(unsigned char *tmp, /* input */
   (void) md5len;
 
   MD5pw = Curl_MD5_init(Curl_DIGEST_MD5);
+  if(!MD5pw)
+    return CURLE_OUT_OF_MEMORY;
   Curl_MD5_update(MD5pw, tmp, curlx_uztoui(tmplen));
   Curl_MD5_final(MD5pw, md5sum);
 #endif
+  return CURLE_OK;
 }
 #endif
 

lib/vtls/vtls.h

@@ -108,7 +108,7 @@ void Curl_ssl_delsessionid(struct connectdata *conn, void *ssl_sessionid);
    in */
 int Curl_ssl_random(struct SessionHandle *data, unsigned char *buffer,
                     size_t length);
-void Curl_ssl_md5sum(unsigned char *tmp, /* input */
+CURLcode Curl_ssl_md5sum(unsigned char *tmp, /* input */
                          size_t tmplen,
                          unsigned char *md5sum, /* output */
                          size_t md5len);