add libssh2_scp_recv2 to support large (> 2GB) files on windows

docs/CMakeLists.txt

@@ -119,6 +119,7 @@ set(MAN_PAGES
   libssh2_publickey_remove_ex.3
   libssh2_publickey_shutdown.3
   libssh2_scp_recv.3
+  libssh2_scp_recv2.3
   libssh2_scp_send.3
   libssh2_scp_send64.3
   libssh2_scp_send_ex.3

docs/Makefile.am

@@ -89,6 +89,7 @@ dist_man_MANS = \
 	libssh2_publickey_remove_ex.3 \
 	libssh2_publickey_shutdown.3 \
 	libssh2_scp_recv.3 \
+	libssh2_scp_recv2.3 \
 	libssh2_scp_send.3 \
 	libssh2_scp_send64.3 \
 	libssh2_scp_send_ex.3 \

docs/libssh2_scp_recv.3

@@ -8,6 +8,9 @@ LIBSSH2_CHANNEL *
 libssh2_scp_recv(LIBSSH2_SESSION *session, const char *path, struct stat *sb);
 
 .SH DESCRIPTION
+This function is \fBDEPRECATED\fP. Use \fIlibssh2_scp_recv2(3)\fP
+instead!
+
 \fIsession\fP - Session instance as returned by 
 .BR libssh2_session_init_ex(3)
 

docs/libssh2_scp_recv2.3

@@ -0,0 +1,32 @@
+.TH libssh2_scp_recv2 3 "29 Jun 2015" "libssh2 1.6.1" "libssh2 manual"
+.SH NAME
+libssh2_scp_recv2 - request a remote file via SCP
+.SH SYNOPSIS
+#include <libssh2.h>
+
+LIBSSH2_CHANNEL *
+libssh2_scp_recv2(LIBSSH2_SESSION *session, const char *path, struct_stat *sb);
+
+.SH DESCRIPTION
+\fIsession\fP - Session instance as returned by 
+.BR libssh2_session_init_ex(3)
+
+\fIpath\fP - Full path and filename of file to transfer. That is the remote
+file name.
+
+\fIsb\fP - Populated with remote file's size, mode, mtime, and atime
+
+Request a file from the remote host via SCP.
+.SH RETURN VALUE
+Pointer to a newly allocated LIBSSH2_CHANNEL instance, or NULL on errors.
+.SH ERRORS
+\fILIBSSH2_ERROR_ALLOC\fP -  An internal memory allocation call failed.
+
+\fILIBSSH2_ERROR_SCP_PROTOCOL\fP - 
+
+\fILIBSSH2_ERROR_EAGAIN\fP - Marked for non-blocking I/O but the call would
+block.
+.SH SEE ALSO
+.BR libssh2_session_init_ex(3)
+.BR libssh2_channel_open_ex(3)
+

example/scp.c

@@ -41,7 +41,7 @@ int main(int argc, char *argv[])
     const char *username="username";
     const char *password="password";
     const char *scppath="/tmp/TEST";
-    struct stat fileinfo;
+    libssh2_struct_stat fileinfo;
     int rc;
     off_t got=0;
 
@@ -137,7 +137,7 @@ int main(int argc, char *argv[])
     }
 
     /* Request a file via SCP */
-    channel = libssh2_scp_recv(session, scppath, &fileinfo);
+    channel = libssh2_scp_recv2(session, scppath, &fileinfo);
 
     if (!channel) {
         fprintf(stderr, "Unable to open a session: %d\n",
@@ -151,7 +151,7 @@ int main(int argc, char *argv[])
         int amount=sizeof(mem);
 
         if((fileinfo.st_size -got) < amount) {
-            amount = fileinfo.st_size -got;
+            amount = (int)(fileinfo.st_size -got);
         }
 
         rc = libssh2_channel_read(channel, mem, amount);

example/scp_nonblock.c

@@ -88,16 +88,16 @@ int main(int argc, char *argv[])
     const char *username="username";
     const char *password="password";
     const char *scppath="/tmp/TEST";
-    struct stat fileinfo;
+    libssh2_struct_stat fileinfo;
 #ifdef HAVE_GETTIMEOFDAY
     struct timeval start;
     struct timeval end;
     long time_ms;
 #endif
     int rc;
-    int total = 0;
     int spin = 0;
-    off_t got=0;
+    libssh2_struct_stat_size got = 0;
+    libssh2_struct_stat_size total = 0;
 
 #ifdef WIN32
     WSADATA wsadata;
@@ -207,9 +207,9 @@ int main(int argc, char *argv[])
 #endif
 
     /* Request a file via SCP */
-    fprintf(stderr, "libssh2_scp_recv()!\n");
+    fprintf(stderr, "libssh2_scp_recv2()!\n");
     do {
-        channel = libssh2_scp_recv(session, scppath, &fileinfo);
+        channel = libssh2_scp_recv2(session, scppath, &fileinfo);
 
         if (!channel) {
             if(libssh2_session_last_errno(session) != LIBSSH2_ERROR_EAGAIN) {
@@ -235,7 +235,7 @@ int main(int argc, char *argv[])
             int amount=sizeof(mem);
 
             if ((fileinfo.st_size -got) < amount) {
-                amount = fileinfo.st_size - got;
+                amount = (int)(fileinfo.st_size - got);
             }
 
             /* loop until we block */
@@ -262,10 +262,10 @@ int main(int argc, char *argv[])
     gettimeofday(&end, NULL);
 
     time_ms = tvdiff(end, start);
-    fprintf(stderr, "Got %d bytes in %ld ms = %.1f bytes/sec spin: %d\n", total,
+    fprintf(stderr, "Got " LIBSSH2_STRUCT_STAT_SIZE_FORMAT " bytes in %ld ms = %.1f bytes/sec spin: %d\n", total,
-           time_ms, total/(time_ms/1000.0), spin );
+           time_ms, total/(time_ms/1000.0), spin);
 #else
-    fprintf(stderr, "Got %d bytes spin: %d\n", total, spin);
+    fprintf(stderr, "Got " LIBSSH2_STRUCT_STAT_SIZE_FORMAT " bytes spin: %d\n", total, spin);
 #endif
 
     libssh2_channel_free(channel);

example/ssh2.c

@@ -243,7 +243,7 @@ int main(int argc, char *argv[])
 
     /* Other channel types are supported via:
      * libssh2_scp_send()
-     * libssh2_scp_recv()
+     * libssh2_scp_recv2()
      * libssh2_channel_direct_tcpip()
      */
 

example/ssh2_agent.c

@@ -217,7 +217,7 @@ int main(int argc, char *argv[])
 
     /* Other channel types are supported via:
      * libssh2_scp_send()
-     * libssh2_scp_recv()
+     * libssh2_scp_recv2()
      * libssh2_channel_direct_tcpip()
      */
 

include/libssh2.h

@@ -145,6 +145,67 @@ typedef int libssh2_socket_t;
 #define LIBSSH2_INVALID_SOCKET -1
 #endif /* WIN32 */
 
+/*
+ * Determine whether there is small or large file support on windows.
+ */
+
+#if defined(_MSC_VER) && !defined(_WIN32_WCE)
+#  if (_MSC_VER >= 900) && (_INTEGRAL_MAX_BITS >= 64)
+#    define USE_WIN32_LARGE_FILES
+#  else
+#    define USE_WIN32_SMALL_FILES
+#  endif
+#endif
+
+#if defined(__MINGW32__) && !defined(USE_WIN32_LARGE_FILES)
+#  define USE_WIN32_LARGE_FILES
+#endif
+
+#if defined(__WATCOMC__) && !defined(USE_WIN32_LARGE_FILES)
+#  define USE_WIN32_LARGE_FILES
+#endif
+
+#if defined(__POCC__)
+#  undef USE_WIN32_LARGE_FILES
+#endif
+
+#if defined(_WIN32) && !defined(USE_WIN32_LARGE_FILES) && !defined(USE_WIN32_SMALL_FILES)
+#  define USE_WIN32_SMALL_FILES
+#endif
+
+/*
+ * Large file (>2Gb) support using WIN32 functions.
+ */
+
+#ifdef USE_WIN32_LARGE_FILES
+#  include <io.h>
+#  include <sys/types.h>
+#  include <sys/stat.h>
+#  define LIBSSH2_STRUCT_STAT_SIZE_FORMAT    "%I64d"
+typedef struct _stati64 libssh2_struct_stat;
+typedef __int64 libssh2_struct_stat_size;
+#endif
+
+/*
+ * Small file (<2Gb) support using WIN32 functions.
+ */
+
+#ifdef USE_WIN32_SMALL_FILES
+#  include <sys/types.h>
+#  include <sys/stat.h>
+#  ifndef _WIN32_WCE
+#    define LIBSSH2_STRUCT_STAT_SIZE_FORMAT    "%d"
+typedef struct _stat libssh2_struct_stat;
+typedef off_t libssh2_struct_stat_size;
+#  endif
+#endif
+
+#ifndef LIBSSH2_STRUCT_STAT_SIZE_FORMAT
+#  define LIBSSH2_STRUCT_STAT_SIZE_FORMAT      "%zd"
+typedef struct stat libssh2_struct_stat;
+typedef off_t libssh2_struct_stat_size;
+#endif
+
 /* Part of every banner, user specified or not */
 #define LIBSSH2_SSH_BANNER                  "SSH-2.0-libssh2_" LIBSSH2_VERSION
 
@@ -805,9 +866,14 @@ LIBSSH2_API int libssh2_channel_close(LIBSSH2_CHANNEL *channel);
 LIBSSH2_API int libssh2_channel_wait_closed(LIBSSH2_CHANNEL *channel);
 LIBSSH2_API int libssh2_channel_free(LIBSSH2_CHANNEL *channel);
 
+/* libssh2_scp_recv is DEPRECATED, do not use! */
 LIBSSH2_API LIBSSH2_CHANNEL *libssh2_scp_recv(LIBSSH2_SESSION *session,
                                               const char *path,
                                               struct stat *sb);
+/* Use libssh2_scp_recv2 for large (> 2GB) file support on windows */
+LIBSSH2_API LIBSSH2_CHANNEL *libssh2_scp_recv2(LIBSSH2_SESSION *session,
+                                               const char *path,
+                                               libssh2_struct_stat *sb);
 LIBSSH2_API LIBSSH2_CHANNEL *libssh2_scp_send_ex(LIBSSH2_SESSION *session,
                                                  const char *path, int mode,
                                                  size_t size, long mtime,

nw/GNUmakefile

@@ -545,6 +545,7 @@ endif
 	@echo $(DL)  libssh2_knownhost_readfile,$(DL) >> $@
 	@echo $(DL)  libssh2_knownhost_writefile,$(DL) >> $@
 	@echo $(DL)  libssh2_scp_recv,$(DL) >> $@
+	@echo $(DL)  libssh2_scp_recv2,$(DL) >> $@
 	@echo $(DL)  libssh2_scp_send64,$(DL) >> $@
 	@echo $(DL)  libssh2_scp_send_ex,$(DL) >> $@
 	@echo $(DL)  libssh2_session_abstract,$(DL) >> $@

src/CMakeLists.txt

@@ -38,6 +38,7 @@ include(CheckSymbolExists)
 include(CheckFunctionExistsMayNeedLibrary)
 include(CheckIncludeFiles)
 include(CheckTypeSize)
+include(CheckSymbolExists)
 include(CheckNonblockingSocketSupport)
 include(SocketLibraries)
 
@@ -281,6 +282,10 @@ if(HAVE_STDLIB_H)
 else()
   check_function_exists(strtoll HAVE_STRTOLL)
 endif()
+if (NOT HAVE_STRTOLL)
+  # Try _strtoi64 if strtoll isn't available
+  check_symbol_exists(_strtoi64 stdlib.h HAVE_STRTOI64)
+endif()
 check_symbol_exists(snprintf stdio.h HAVE_SNPRINTF)
 
 if(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin" OR

src/libssh2_config_cmake.h.in

@@ -64,6 +64,7 @@
 #cmakedefine HAVE_SELECT
 #cmakedefine HAVE_SOCKET
 #cmakedefine HAVE_STRTOLL
+#cmakedefine HAVE_STRTOI64
 #cmakedefine HAVE_SNPRINTF
 
 /* OpenSSL functions */

src/libssh2_priv.h

@@ -779,7 +779,7 @@ struct _LIBSSH2_SESSION
     int sftpInit_sent; /* number of bytes from the buffer that have been
                           sent */
 
-    /* State variables used in libssh2_scp_recv() */
+    /* State variables used in libssh2_scp_recv() / libssh_scp_recv2() */
     libssh2_nonblocking_states scpRecv_state;
     unsigned char *scpRecv_command;
     size_t scpRecv_command_len;
@@ -790,6 +790,9 @@ struct _LIBSSH2_SESSION
     /* we have the type and we can parse such numbers */
     long long scpRecv_size;
 #define scpsize_strtol strtoll
+#elif defined(HAVE_STRTOI64)
+    __int64 scpRecv_size;
+#define scpsize_strtol _strtoi64
 #else
     long scpRecv_size;
 #define scpsize_strtol strtol

src/scp.c

@@ -268,7 +268,7 @@ shell_quotearg(const char *path, unsigned char *buf,
  *
  */
 static LIBSSH2_CHANNEL *
-scp_recv(LIBSSH2_SESSION * session, const char *path, struct stat * sb)
+scp_recv(LIBSSH2_SESSION * session, const char *path, libssh2_struct_stat * sb)
 {
     int cmd_len;
     int rc;
@@ -724,7 +724,7 @@ scp_recv(LIBSSH2_SESSION * session, const char *path, struct stat * sb)
     }
 
     if (sb) {
-        memset(sb, 0, sizeof(struct stat));
+        memset(sb, 0, sizeof(libssh2_struct_stat));
 
         sb->st_mtime = session->scpRecv_mtime;
         sb->st_atime = session->scpRecv_atime;
@@ -759,11 +759,47 @@ scp_recv(LIBSSH2_SESSION * session, const char *path, struct stat * sb)
 /*
  * libssh2_scp_recv
  *
- * Open a channel and request a remote file via SCP
+ * DEPRECATED
+ *
+ * Open a channel and request a remote file via SCP.  This receives files larger
+ * than 2 GB, but is unable to report the proper size on platforms where the
+ * st_size member of struct stat is limited to 2 GB (e.g. windows).
  *
  */
 LIBSSH2_API LIBSSH2_CHANNEL *
 libssh2_scp_recv(LIBSSH2_SESSION *session, const char *path, struct stat * sb)
+{
+    LIBSSH2_CHANNEL *ptr;
+
+    /* scp_recv uses libssh2_struct_stat, so pass one if the caller gave us a struct to populate... */
+    libssh2_struct_stat sb_intl;
+    libssh2_struct_stat *sb_ptr;
+    sb_ptr = sb ? &sb_intl : NULL;
+
+    BLOCK_ADJUST_ERRNO(ptr, session, scp_recv(session, path, sb_ptr));
+
+    /* ...and populate the caller's with as much info as fits. */
+    if (sb) {
+        memset(sb, 0, sizeof(struct stat));
+
+        sb->st_mtime = sb_intl.st_mtime;
+        sb->st_atime = sb_intl.st_atime;
+        sb->st_size = (off_t)sb_intl.st_size;
+        sb->st_mode = sb_intl.st_mode;
+    }
+
+    return ptr;
+}
+
+/*
+ * libssh2_scp_recv2
+ *
+ * Open a channel and request a remote file via SCP.  This supports files > 2GB
+ * on platforms that support it.
+ *
+ */
+LIBSSH2_API LIBSSH2_CHANNEL *
+libssh2_scp_recv2(LIBSSH2_SESSION *session, const char *path, libssh2_struct_stat * sb)
 {
     LIBSSH2_CHANNEL *ptr;
     BLOCK_ADJUST_ERRNO(ptr, session, scp_recv(session, path, sb));