Pass private-key to OpenSSL as a filename with BIO_new_file().

This keeps all FILE* handling on the OpenSSL side of the DLL boundary avoiding crashes on Windows while removing the need for libssh2 to read the private key file into memory.  This is now done by OpenSSL which is likely to do a better job of it.

src/hostkey.c

@@ -114,7 +114,6 @@ hostkey_method_ssh_rsa_initPEM(LIBSSH2_SESSION * session,
                                void **abstract)
 {
     libssh2_rsa_ctx *rsactx;
-    FILE *fp;
     int ret;
 
     if (*abstract) {
@@ -122,13 +121,7 @@ hostkey_method_ssh_rsa_initPEM(LIBSSH2_SESSION * session,
         *abstract = NULL;
     }
 
-    fp = fopen(privkeyfile, "r");
+    ret = _libssh2_rsa_new_private(&rsactx, session, privkeyfile, passphrase);
-    if (!fp) {
-        return -1;
-    }
-
-    ret = _libssh2_rsa_new_private(&rsactx, session, fp, passphrase);
-    fclose(fp);
     if (ret) {
         return -1;
     }
@@ -296,7 +289,6 @@ hostkey_method_ssh_dss_initPEM(LIBSSH2_SESSION * session,
                                void **abstract)
 {
     libssh2_dsa_ctx *dsactx;
-    FILE *fp;
     int ret;
 
     if (*abstract) {
@@ -304,13 +296,7 @@ hostkey_method_ssh_dss_initPEM(LIBSSH2_SESSION * session,
         *abstract = NULL;
     }
 
-    fp = fopen(privkeyfile, "r");
+    ret = _libssh2_dsa_new_private(&dsactx, session, privkeyfile, passphrase);
-    if (!fp) {
-        return -1;
-    }
-
-    ret = _libssh2_dsa_new_private(&dsactx, session, fp, passphrase);
-    fclose(fp);
     if (ret) {
         return -1;
     }

src/libgcrypt.c

@@ -149,8 +149,9 @@ _libssh2_dsa_new(libssh2_dsa_ctx ** dsactx,
 int
 _libssh2_rsa_new_private(libssh2_rsa_ctx ** rsa,
                          LIBSSH2_SESSION * session,
-                         FILE * fp, unsigned const char *passphrase)
+                         const char *filename, unsigned const char *passphrase)
 {
+    FILE *fp;
     unsigned char *data, *save_data;
     unsigned int datalen;
     int ret;
@@ -159,10 +160,16 @@ _libssh2_rsa_new_private(libssh2_rsa_ctx ** rsa,
 
     (void) passphrase;
 
+    fp = fopen(filename, "r");
+    if (!fp) {
+        return -1;
+    }
+
     ret = _libssh2_pem_parse(session,
                              "-----BEGIN RSA PRIVATE KEY-----",
                              "-----END RSA PRIVATE KEY-----",
                              fp, &data, &datalen);
+    fclose(fp)
     if (ret) {
         return -1;
     }
@@ -244,8 +251,9 @@ _libssh2_rsa_new_private(libssh2_rsa_ctx ** rsa,
 int
 _libssh2_dsa_new_private(libssh2_dsa_ctx ** dsa,
                          LIBSSH2_SESSION * session,
-                         FILE * fp, unsigned const char *passphrase)
+                         const char *filename, unsigned const char *passphrase)
 {
+    FILE *fp;
     unsigned char *data, *save_data;
     unsigned int datalen;
     int ret;
@@ -254,10 +262,16 @@ _libssh2_dsa_new_private(libssh2_dsa_ctx ** dsa,
 
     (void) passphrase;
 
+    fp = fopen(filename);
+    if (!fp) {
+        return -1;
+    }
+
     ret = _libssh2_pem_parse(session,
                              "-----BEGIN DSA PRIVATE KEY-----",
                              "-----END DSA PRIVATE KEY-----",
                              fp, &data, &datalen);
+    fclose(fp);
     if (ret) {
         return -1;
     }

src/libgcrypt.h

@@ -112,7 +112,8 @@ int _libssh2_rsa_new(libssh2_rsa_ctx ** rsa,
                      const unsigned char *coeffdata, unsigned long coefflen);
 int _libssh2_rsa_new_private(libssh2_rsa_ctx ** rsa,
                              LIBSSH2_SESSION * session,
-                             FILE * fp, unsigned const char *passphrase);
+                             const char *filename,
+                             unsigned const char *passphrase);
 int _libssh2_rsa_sha1_verify(libssh2_rsa_ctx * rsa,
                              const unsigned char *sig,
                              unsigned long sig_len,
@@ -140,7 +141,8 @@ int _libssh2_dsa_new(libssh2_dsa_ctx ** dsa,
                      const unsigned char *x, unsigned long x_len);
 int _libssh2_dsa_new_private(libssh2_dsa_ctx ** dsa,
                              LIBSSH2_SESSION * session,
-                             FILE * fp, unsigned const char *passphrase);
+                             const char *filename,
+                             unsigned const char *passphrase);
 int _libssh2_dsa_sha1_verify(libssh2_dsa_ctx * dsa,
                              const unsigned char *sig,
                              const unsigned char *m, unsigned long m_len);

src/openssl.c

@@ -43,9 +43,6 @@
 #define EVP_MAX_BLOCK_LENGTH 32
 #endif
 
-/* Ridiculously large key-file size cap (512KB) */
-#define MAX_KEY_FILE_LENGTH 524288
-
 int
 _libssh2_rsa_new(libssh2_rsa_ctx ** rsa,
                  const unsigned char *edata,
@@ -215,61 +212,21 @@ passphrase_cb(char *buf, int size, int rwflag, char *passphrase)
     return passphrase_len;
 }
 
-static int
-read_file_into_string(char ** key, LIBSSH2_SESSION * session, FILE * fp)
-{
-    long size;
-    size_t read;
-
-    *key = NULL;
-
-    fseek(fp, 0, SEEK_END);
-    size = ftell(fp);
-    if (size < 0) {
-        return -1;
-    }
-    fseek(fp, 0, SEEK_SET);
-
-    size *= sizeof(char);
-    if (size > MAX_KEY_FILE_LENGTH) {
-        return -1;
-    }
-
-    *key = LIBSSH2_ALLOC(session, size + 1);
-    if (!*key) {
-        return -1;
-    }
-
-    read = fread(*key, 1, size, fp);
-    if (read != (size_t) size) {
-        LIBSSH2_FREE(session, *key);
-        return -1;
-    }
-
-    (*key)[size] = '\0';
-    return 0;
-}
-
 typedef void * (*pem_read_bio_func)(BIO *, void **, pem_password_cb *,
                                     void * u);
 
 static int
 read_private_key_from_file(void ** key_ctx, LIBSSH2_SESSION * session,
                            pem_read_bio_func read_private_key,
-                           FILE * fp, unsigned const char *passphrase)
+                           const char * filename,
+                           unsigned const char *passphrase)
 {
-    char * key;
     BIO * bp;
 
     *key_ctx = NULL;
 
-    if(read_file_into_string(&key, session, fp)) {
+    bp = BIO_new_file(filename, "r");
-        return -1;
-    }
-
-    bp = BIO_new_mem_buf(key, -1);
     if (!bp) {
-        LIBSSH2_FREE(session, key);
         return -1;
     }
 
@@ -277,14 +234,13 @@ read_private_key_from_file(void ** key_ctx, LIBSSH2_SESSION * session,
                                 (void *) passphrase);
 
     BIO_free(bp);
-    LIBSSH2_FREE(session, key);
     return (*key_ctx) ? 0 : -1;
 }
 
 int
 _libssh2_rsa_new_private(libssh2_rsa_ctx ** rsa,
                          LIBSSH2_SESSION * session,
-                         FILE * fp, unsigned const char *passphrase)
+                         const char *filename, unsigned const char *passphrase)
 {
     pem_read_bio_func read_rsa =
         (pem_read_bio_func) &PEM_read_bio_RSAPrivateKey;
@@ -297,14 +253,14 @@ _libssh2_rsa_new_private(libssh2_rsa_ctx ** rsa,
         OpenSSL_add_all_ciphers();
     }
 
-    return read_private_key_from_file((void **) rsa, session, read_rsa, fp,
+    return read_private_key_from_file((void **) rsa, session, read_rsa,
-                                      passphrase);
+                                      filename, passphrase);
 }
 
 int
 _libssh2_dsa_new_private(libssh2_dsa_ctx ** dsa,
                          LIBSSH2_SESSION * session,
-                         FILE * fp, unsigned const char *passphrase)
+                         const char *filename, unsigned const char *passphrase)
 {
     pem_read_bio_func read_dsa =
         (pem_read_bio_func) &PEM_read_bio_DSAPrivateKey;
@@ -317,8 +273,8 @@ _libssh2_dsa_new_private(libssh2_dsa_ctx ** dsa,
         OpenSSL_add_all_ciphers();
     }
 
-    return read_private_key_from_file((void **) dsa, session, read_dsa, fp,
+    return read_private_key_from_file((void **) dsa, session, read_dsa,
-                                      passphrase);
+                                      filename, passphrase);
 }
 
 int

src/openssl.h

@@ -148,7 +148,8 @@ int _libssh2_rsa_new(libssh2_rsa_ctx ** rsa,
                      const unsigned char *coeffdata, unsigned long coefflen);
 int _libssh2_rsa_new_private(libssh2_rsa_ctx ** rsa,
                              LIBSSH2_SESSION * session,
-                             FILE * fp, unsigned const char *passphrase);
+                             const char *filename,
+                             unsigned const char *passphrase);
 int _libssh2_rsa_sha1_verify(libssh2_rsa_ctx * rsa,
                              const unsigned char *sig,
                              unsigned long sig_len,
@@ -176,7 +177,8 @@ int _libssh2_dsa_new(libssh2_dsa_ctx ** dsa,
                      const unsigned char *x, unsigned long x_len);
 int _libssh2_dsa_new_private(libssh2_dsa_ctx ** dsa,
                              LIBSSH2_SESSION * session,
-                             FILE * fp, unsigned const char *passphrase);
+                             const char *filename,
+                             unsigned const char *passphrase);
 int _libssh2_dsa_sha1_verify(libssh2_dsa_ctx * dsactx,
                              const unsigned char *sig,
                              const unsigned char *m, unsigned long m_len);