generic-library/curl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

David J Meyer's large file support.

Browse Source
This commit is contained in:
Daniel Stenberg 2004-01-05 22:29:29 +00:00
parent 41c6f68d94
commit b60e0fa97e
27 changed files with 290 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/examples/ftpupload.c
View File

@ -73,7 +73,7 @@ int main(int argc, char **argv)
curl_easy_setopt(curl, CURLOPT_READDATA, hd_src);
/* and give the size of the upload (optional) */
curl_easy_setopt(curl, CURLOPT_INFILESIZE, file_info.st_size);
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, file_info.st_size);
/* Now run off and do what you've been told! */
res = curl_easy_perform(curl);

5
docs/examples/httpput.c
View File

@ -85,9 +85,8 @@ int main(int argc, char **argv)
/* now specify which file to upload */
curl_easy_setopt(curl, CURLOPT_READDATA, hd_src);
/* and give the size of the upload, make sure that we don't accidentally
pass a larger variable type than "long". */
curl_easy_setopt(curl, CURLOPT_INFILESIZE, (long) file_info.st_size);
/* and give the size of the upload */
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, file_info.st_size);
/* Now run off and do what you've been told! */
res = curl_easy_perform(curl);

4
docs/libcurl-the-guide
View File

@ -298,9 +298,9 @@ Upload Data to a Remote Site
A few protocols won't behave properly when uploads are done without any prior
knowledge of the expected file size. So, set the upload file size using the
CURLOPT_INFILESIZE for all known file sizes like this[1]:
CURLOPT_INFILESIZE_LARGE for all known file sizes like this[1]:
curl_easy_setopt(easyhandle, CURLOPT_INFILESIZE, file_size);
curl_easy_setopt(easyhandle, CURLOPT_INFILESIZE_LARGE, file_size);
When you call curl_easy_perform() this time, it'll perform all the necessary
operations and when it has invoked the upload it'll call your supplied

22
docs/libcurl/curl_easy_setopt.3
View File

@ -602,6 +602,9 @@ techniques).
.IP CURLOPT_RESUME_FROM
Pass a long as parameter. It contains the offset in number of bytes that you
want the transfer to start from.
.IP CURLOPT_RESUME_FROM_LARGE
Pass an off_t as parameter. It contains the offset in number of bytes that you
want the transfer to start from.
.IP CURLOPT_CUSTOMREQUEST
Pass a pointer to a zero terminated string as parameter. It will be user
instead of GET or HEAD when doing a HTTP request, or instead of LIST or NLST
@ -628,16 +631,29 @@ output. This is only relevant for protocols that have separate header and body
parts. On HTTP(S) servers, this will make libcurl do a HEAD request.
.IP CURLOPT_INFILESIZE
When uploading a file to a remote site, this option should be used to tell
libcurl what the expected size of the infile is.
libcurl what the expected size of the infile is. This value should be passed
as a long. See also \fICURLOPT_INFILESIZE_LARGE\fP.
.IP CURLOPT_INFILESIZE_LARGE
When uploading a file to a remote site, this option should be used to tell
libcurl what the expected size of the infile is. This value should be passed
as an off_t.
.IP CURLOPT_UPLOAD
A non-zero parameter tells the library to prepare for an upload. The
\fICURLOPT_READDATA\fP and \fICURLOPT_INFILESIZE\fP are also interesting for
uploads.
\fICURLOPT_READDATA\fP and \fICURLOPT_INFILESIZE_LARGE\fP are also interesting
for uploads.
.IP CURLOPT_MAXFILESIZE
Pass a long as parameter. This allows you to specify the maximum size (in
bytes) of a file to download. If the file requested is larger than this value,
the transfer will not start and CURLE_FILESIZE_EXCEEDED will be returned.
NOTE: The file size is not always known prior to download, and for such files
this option has no effect even if the file transfer ends up being larger than
this given limit. This concerns both FTP and HTTP transfers.
.IP CURLOPT_MAXFILESIZE_LARGE
Pass an off_t as parameter. This allows you to specify the maximum size (in
bytes) of a file to download. If the file requested is larger than this value,
the transfer will not start and CURLE_FILESIZE_EXCEEDED will be returned.
NOTE: The file size is not always known prior to download, and for such files
this option has no effect even if the file transfer ends up being larger than
this given limit. This concerns both FTP and HTTP transfers.

36
include/curl/curl.h
View File

@ -270,6 +270,7 @@ typedef enum {
#define CURLOPTTYPE_LONG 0
#define CURLOPTTYPE_OBJECTPOINT 10000
#define CURLOPTTYPE_FUNCTIONPOINT 20000
#define CURLOPTTYPE_OFF_T 30000
/* name is uppercase CURLOPT_<name>,
type is one of the defined CURLOPTTYPE_<type>
@ -300,6 +301,7 @@ typedef enum {
#define LONG CURLOPTTYPE_LONG
#define OBJECTPOINT CURLOPTTYPE_OBJECTPOINT
#define FUNCTIONPOINT CURLOPTTYPE_FUNCTIONPOINT
#define OFF_T CURLOPTTYPE_OFF_T
#define CINIT(name,type,number) CURLOPT_/**/name = type + number
#endif
@ -356,7 +358,12 @@ typedef enum {
/* If the CURLOPT_INFILE is used, this can be used to inform libcurl about
* how large the file being sent really is. That allows better error
* checking and better verifies that the upload was succcessful. -1 means
* unknown size. */
* unknown size.
*
* For large file support, there is also a _LARGE version of the key
* which takes an off_t type, allowing platforms with larger off_t
* sizes to handle larger files. See below for INFILESIZE_LARGE.
*/
CINIT(INFILESIZE, LONG, 14),
/* POST input fields. */
@ -384,7 +391,12 @@ typedef enum {
/* Set the "low speed time" */
CINIT(LOW_SPEED_TIME, LONG, 20),
/* Set the continuation offset */
/* Set the continuation offset.
*
* Note there is also a _LARGE version of this key which uses
* off_t types, allowing for large file offsets on platforms which
* use larger-than-32-bit off_t's. Look below for RESUME_FROM_LARGE.
*/
CINIT(RESUME_FROM, LONG, 21),
/* Set cookie in request: */
@ -699,9 +711,27 @@ typedef enum {
CINIT(IPRESOLVE, LONG, 113),
/* Set this option to limit the size of a file that will be downloaded from
an HTTP or FTP server. */
an HTTP or FTP server.
Note there is also _LARGE version which adds large file support for
platforms which have larger off_t sizes. See MAXFILESIZE_LARGE below. */
CINIT(MAXFILESIZE, LONG, 114),
/* See the comment for INFILESIZE above, but in short, specifies
* the size of the file being uploaded. -1 means unknown.
*/
CINIT(INFILESIZE_LARGE, OFF_T, 115),
/* Sets the continuation offset. There is also a LONG version of this;
* look above for RESUME_FROM.
*/
CINIT(RESUME_FROM_LARGE, OFF_T, 116),
/* Sets the maximum size of data that will be downloaded from
* an HTTP or FTP server. See MAXFILESIZE above for the LONG version.
*/
CINIT(MAXFILESIZE_LARGE, OFF_T, 117),
/* Set this option to the file name of your .netrc file you want libcurl
to parse (using the CURLOPT_NETRC option). If not set, libcurl will do
a poor attempt to find the user's home directory and check for a .netrc

3
lib/Makefile.am
View File

@ -74,7 +74,8 @@ memdebug.h inet_ntoa_r.h http_chunks.c http_chunks.h strtok.c strtok.h \
connect.c connect.h llist.c llist.h hash.c hash.h multi.c \
content_encoding.c content_encoding.h share.c share.h http_digest.c \
md5.c md5.h http_digest.h http_negotiate.c http_negotiate.h \
http_ntlm.c http_ntlm.h ca-bundle.h inet_pton.c inet_pton.h
http_ntlm.c http_ntlm.h ca-bundle.h inet_pton.c inet_pton.h \
strtoofft.c strtoofft.h
noinst_HEADERS = setup.h transfer.h

1
lib/Makefile.b32
View File

@ -61,6 +61,7 @@ SOURCES = \
easy.c \
strequal.c \
strtok.c \
strtoofft.c \
connect.c \
hash.c \
share.c \

1
lib/Makefile.b32.resp
View File

@ -30,6 +30,7 @@
+easy.obj &
+strequal.obj &
+strtok.obj &
+strtoofft.obj &
+connect.obj &
+hash.obj &
+share.obj &

5
lib/Makefile.m32
View File

@ -47,7 +47,7 @@ libcurl_a_SOURCES = arpa_telnet.h file.c netrc.h timeval.c base64.c \
strtok.c connect.c hash.c llist.c multi.c share.c share.h \
content_encoding.h content_encoding.c http_digest.h http_digest.c \
http_negotiate.c http_negotiate.h http_ntlm.c http_ntlm.h md5.h \
md5.c
md5.c strtoofft.c
libcurl_a_OBJECTS = file.o timeval.o base64.o hostip.o progress.o \
formdata.o cookie.o http.o sendf.o ftp.o url.o dict.o if2ip.o \
@ -55,7 +55,8 @@ libcurl_a_OBJECTS = file.o timeval.o base64.o hostip.o progress.o \
getenv.o escape.o mprintf.o telnet.o netrc.o getinfo.o \
strequal.o easy.o security.o krb4.o memdebug.o http_chunks.o \
strtok.o connect.o hash.o llist.o multi.o share.o \
content_encoding.o http_digest.o http_negotiate.o http_ntlm.o md5.o
content_encoding.o http_digest.o http_negotiate.o http_ntlm.o md5.o \
strtoofft.o
LIBRARIES = $(libcurl_a_LIBRARIES)
SOURCES = $(libcurl_a_SOURCES)

5
lib/Makefile.riscos
View File

@ -11,7 +11,7 @@ objs = o.base64 o.connect o.cookie o.dict \
o.memdebug o.mprintf o.netrc o.progress \
o.security o.sendf o.speedcheck o.ssluse \
o.strequal o.strtok o.telnet o.timeval \
o.transfer o.url o.version
o.transfer o.url o.version o.strtoofft
# Compile options:
linkopts = -o libcurl
@ -112,6 +112,9 @@ o.strequal: c.strequal
o.strtok: c.strtok
gcc $(compileropts) -c -o strtok.o c.strtok
o.strtoofft: c.strtoofft
gcc $(compileropts) -c -o strtoofft.o c.strtoofft
o.telnet: c.telnet
gcc $(compileropts) -c -o telnet.o c.telnet

1
lib/Makefile.vc6
View File

@ -218,6 +218,7 @@ X_OBJS= \
$(DIROBJ)\easy.obj \
$(DIROBJ)\strequal.obj \
$(DIROBJ)\strtok.obj \
$(DIROBJ)\strtoofft.obj \
$(DIROBJ)\connect.obj \
$(DIROBJ)\hash.obj \
$(DIROBJ)\llist.obj \

3
lib/config-riscos.h
View File

@ -287,6 +287,9 @@
/* Define if you have the `strtok_r' function. */
#undef HAVE_STRTOK_R
/* Define if you have the `strtoll' function. */
#undef HAVE_STRTOLL
/* Define if you have the <sys/param.h> header file. */
#undef HAVE_SYS_PARAM_H

3
lib/config-vms.h
View File

@ -363,6 +363,9 @@
/* Define if you have the `strtok_r' function. */
#undef HAVE_STRTOK_R
/* Define if you have the `strtoll' function. */
#undef HAVE_STRTOLL
#define HAVE_MEMORY_H 1
#define HAVE_FIONBIO 1

2
lib/dict.c
View File

@ -92,7 +92,7 @@ CURLcode Curl_dict(struct connectdata *conn)
int sockfd = conn->sock[FIRSTSOCKET];
char *path = conn->path;
long *bytecount = &conn->bytecount;
off_t *bytecount = &conn->bytecount;
if(conn->bits.user_passwd) {
/* AUTH is missing */

8
lib/easy.c
View File

@ -202,6 +202,7 @@ CURLcode curl_easy_setopt(CURL *curl, CURLoption tag, ...)
func_T param_func = (func_T)0;
long param_long = 0;
void *param_obj = NULL;
off_t param_offset = 0;
struct SessionHandle *data = curl;
CURLcode ret=CURLE_FAILED_INIT;
@ -224,9 +225,14 @@ CURLcode curl_easy_setopt(CURL *curl, CURLoption tag, ...)
param_obj = va_arg(arg, void *);
ret = Curl_setopt(data, tag, param_obj);
}
else {
else if(tag < CURLOPTTYPE_OFF_T) {
/* This is a function pointer type */
param_func = va_arg(arg, func_T );
ret = Curl_setopt(data, tag, param_func);
} else {
/* This is an off_t type */
param_offset = va_arg(arg, off_t);
ret = Curl_setopt(data, tag, param_offset);
}
va_end(arg);

8
lib/file.c
View File

@ -163,12 +163,12 @@ CURLcode Curl_file(struct connectdata *conn)
*/
CURLcode res = CURLE_OK;
struct stat statbuf;
unsigned long expected_size=0;
off_t expected_size=0;
bool fstated=FALSE;
ssize_t nread;
struct SessionHandle *data = conn->data;
char *buf = data->state.buffer;
int bytecount = 0;
off_t bytecount = 0;
struct timeval start = Curl_tvnow();
struct timeval now = start;
int fd;
@ -188,7 +188,7 @@ CURLcode Curl_file(struct connectdata *conn)
date. */
if(data->set.no_body && data->set.include_header && fstated) {
CURLcode result;
sprintf(buf, "Content-Length: %lu\r\n", expected_size);
sprintf(buf, "Content-Length: %Od\r\n", expected_size);
result = Curl_client_write(data, CLIENTWRITE_BOTH, buf, 0);
if(result)
return result;
@ -217,7 +217,7 @@ CURLcode Curl_file(struct connectdata *conn)
}
/* Added by Dolbneff A.V & Spiridonoff A.V */
if (conn->resume_from <= (long)expected_size)
if (conn->resume_from <= expected_size)
expected_size -= conn->resume_from;
else
/* Is this error code suitable in such situation? */

90
lib/ftp.c
View File

@ -82,6 +82,7 @@
#include "krb4.h"
#endif
#include "strtoofft.h"
#include "strequal.h"
#include "ssluse.h"
#include "connect.h"
@ -284,9 +285,16 @@ CURLcode Curl_GetFTPResponse(ssize_t *nreadp, /* return number of bytes read */
*/
if(ftp->cache) {
/* we had data in the "cache", copy that instead of doing an actual
read */
memcpy(ptr, ftp->cache, ftp->cache_size);
gotbytes = ftp->cache_size;
* read
*
* Dave Meyer, December 2003:
* ftp->cache_size is cast to int here. This should be safe,
* because it would have been populated with something of size
* int to begin with, even though its datatype may be larger
* than an int.
*/
memcpy(ptr, ftp->cache, (int)ftp->cache_size);
gotbytes = (int)ftp->cache_size;
free(ftp->cache); /* free the cache */
ftp->cache = NULL; /* clear the pointer */
ftp->cache_size = 0; /* zero the size just in case */
@ -364,9 +372,9 @@ CURLcode Curl_GetFTPResponse(ssize_t *nreadp, /* return number of bytes read */
already! Cleverly figured out by Eric Lavigne in December
2001. */
ftp->cache_size = gotbytes - i;
ftp->cache = (char *)malloc(ftp->cache_size);
ftp->cache = (char *)malloc((int)ftp->cache_size);
if(ftp->cache)
memcpy(ftp->cache, line_start, ftp->cache_size);
memcpy(ftp->cache, line_start, (int)ftp->cache_size);
else
return CURLE_OUT_OF_MEMORY; /**BANG**/
}
@ -712,8 +720,8 @@ CURLcode Curl_ftp_done(struct connectdata *conn)
if((-1 != data->set.infilesize) &&
(data->set.infilesize != *ftp->bytecountp) &&
!data->set.crlf) {
failf(data, "Uploaded unaligned file size (%d out of %d bytes)",
*ftp->bytecountp, data->set.infilesize);
failf(data, "Uploaded unaligned file size (%Od out of %Od bytes)",
*ftp->bytecountp, data->set.infilesize);
conn->bits.close = TRUE; /* close this connection since we don't
know what state this error leaves us in */
return CURLE_PARTIAL_FILE;
@ -722,7 +730,8 @@ CURLcode Curl_ftp_done(struct connectdata *conn)
else {
if((-1 != conn->size) && (conn->size != *ftp->bytecountp) &&
(conn->maxdownload != *ftp->bytecountp)) {
failf(data, "Received only partial file: %d bytes", *ftp->bytecountp);
failf(data, "Received only partial file: %Od bytes",
*ftp->bytecountp);
conn->bits.close = TRUE; /* close this connection since we don't
know what state this error leaves us in */
return CURLE_PARTIAL_FILE;
@ -912,7 +921,7 @@ static CURLcode ftp_transfertype(struct connectdata *conn,
static
CURLcode ftp_getsize(struct connectdata *conn, char *file,
ssize_t *size)
off_t *size)
{
struct SessionHandle *data = conn->data;
int ftpcode;
@ -927,7 +936,7 @@ CURLcode ftp_getsize(struct connectdata *conn, char *file,
if(ftpcode == 213) {
/* get the size from the ascii string: */
*size = atoi(buf+4);
*size = strtoofft(buf+4, NULL, 0);
}
else
return CURLE_FTP_COULDNT_GET_SIZE;
@ -1651,7 +1660,7 @@ CURLcode Curl_ftp_nextconnect(struct connectdata *conn)
/* the ftp struct is already inited in Curl_ftp_connect() */
struct FTP *ftp = conn->proto.ftp;
long *bytecountp = ftp->bytecountp;
off_t *bytecountp = ftp->bytecountp;
if(data->set.upload) {
@ -1683,7 +1692,7 @@ CURLcode Curl_ftp_nextconnect(struct connectdata *conn)
if(conn->resume_from < 0 ) {
/* we could've got a specified offset from the command line,
but now we know we didn't */
ssize_t gottensize;
off_t gottensize;
if(CURLE_OK != ftp_getsize(conn, ftp->file, &gottensize)) {
failf(data, "Couldn't get remote file size");
@ -1694,7 +1703,7 @@ CURLcode Curl_ftp_nextconnect(struct connectdata *conn)
if(conn->resume_from) {
/* do we still game? */
int passed=0;
off_t passed=0;
/* enable append instead */
data->set.ftp_append = 1;
@ -1702,19 +1711,20 @@ CURLcode Curl_ftp_nextconnect(struct connectdata *conn)
input. If we knew it was a proper file we could've just
fseek()ed but we only have a stream here */
do {
int readthisamountnow = (conn->resume_from - passed);
int actuallyread;
off_t readthisamountnow = (conn->resume_from - passed);
off_t actuallyread;
if(readthisamountnow > BUFSIZE)
readthisamountnow = BUFSIZE;
actuallyread =
conn->fread(data->state.buffer, 1, readthisamountnow,
conn->fread(data->state.buffer, 1, (size_t)readthisamountnow,
conn->fread_in);
passed += actuallyread;
if(actuallyread != readthisamountnow) {
failf(data, "Could only read %d bytes from the input", passed);
failf(data, "Could only read %Od bytes from the input",
passed);
return CURLE_FTP_COULDNT_USE_REST;
}
}
@ -1781,7 +1791,7 @@ CURLcode Curl_ftp_nextconnect(struct connectdata *conn)
/* When we know we're uploading a specified file, we can get the file
size prior to the actual upload. */
Curl_pgrsSetUploadSize(data, data->set.infilesize);
Curl_pgrsSetUploadSize(data, (double)data->set.infilesize);
result = Curl_Transfer(conn, -1, -1, FALSE, NULL, /* no download */
SECONDARYSOCKET, bytecountp);
@ -1792,18 +1802,18 @@ CURLcode Curl_ftp_nextconnect(struct connectdata *conn)
else if(!data->set.no_body) {
/* Retrieve file or directory */
bool dirlist=FALSE;
long downloadsize=-1;
off_t downloadsize=-1;
if(conn->bits.use_range && conn->range) {
long from, to;
int totalsize=-1;
off_t from, to;
off_t totalsize=-1;
char *ptr;
char *ptr2;
from=strtol(conn->range, &ptr, 0);
from=strtoofft(conn->range, &ptr, 0);
while(ptr && *ptr && (isspace((int)*ptr) || (*ptr=='-')))
ptr++;
to=strtol(ptr, &ptr2, 0);
to=strtoofft(ptr, &ptr2, 0);
if(ptr == ptr2) {
/* we didn't get any digit */
to=-1;
@ -1811,25 +1821,25 @@ CURLcode Curl_ftp_nextconnect(struct connectdata *conn)
if((-1 == to) && (from>=0)) {
/* X - */
conn->resume_from = from;
infof(data, "FTP RANGE %d to end of file\n", from);
infof(data, "FTP RANGE %Od to end of file\n", from);
}
else if(from < 0) {
/* -Y */
totalsize = -from;
conn->maxdownload = -from;
conn->resume_from = from;
infof(data, "FTP RANGE the last %d bytes\n", totalsize);
infof(data, "FTP RANGE the last %Od bytes\n", totalsize);
}
else {
/* X-Y */
totalsize = to-from;
conn->maxdownload = totalsize+1; /* include the last mentioned byte */
conn->resume_from = from;
infof(data, "FTP RANGE from %d getting %d bytes\n", from,
conn->maxdownload);
infof(data, "FTP RANGE from %Od getting %Od bytes\n", from,
conn->maxdownload);
}
infof(data, "range-download from %d to %d, totally %d bytes\n",
from, to, conn->maxdownload);
infof(data, "range-download from %Od to %Od, totally %Od bytes\n",
from, to, conn->maxdownload);
ftp->dont_check = TRUE; /* dont check for successful transfer */
}
@ -1853,7 +1863,7 @@ CURLcode Curl_ftp_nextconnect(struct connectdata *conn)
(data->set.ftp_list_only?"NLST":"LIST"));
}
else {
ssize_t foundsize;
off_t foundsize;
/* Set type to binary (unless specified ASCII) */
result = ftp_transfertype(conn, data->set.ftp_ascii);
@ -1903,7 +1913,7 @@ CURLcode Curl_ftp_nextconnect(struct connectdata *conn)
if(conn->resume_from< 0) {
/* We're supposed to download the last abs(from) bytes */
if(foundsize < -conn->resume_from) {
failf(data, "Offset (%d) was beyond file size (%d)",
failf(data, "Offset (%Od) was beyond file size (%Od)",
conn->resume_from, foundsize);
return CURLE_FTP_BAD_DOWNLOAD_RESUME;
}
@ -1914,7 +1924,7 @@ CURLcode Curl_ftp_nextconnect(struct connectdata *conn)
}
else {
if(foundsize < conn->resume_from) {
failf(data, "Offset (%d) was beyond file size (%d)",
failf(data, "Offset (%Od) was beyond file size (%Od)",
conn->resume_from, foundsize);
return CURLE_FTP_BAD_DOWNLOAD_RESUME;
}
@ -1935,10 +1945,10 @@ CURLcode Curl_ftp_nextconnect(struct connectdata *conn)
}
/* Set resume file transfer offset */
infof(data, "Instructs server to resume from offset %d\n",
infof(data, "Instructs server to resume from offset %Od\n",
conn->resume_from);
FTPSENDF(conn, "REST %d", conn->resume_from);
FTPSENDF(conn, "REST %Od", conn->resume_from);
result = Curl_GetFTPResponse(&nread, conn, &ftpcode);
if(result)
@ -1976,7 +1986,7 @@ CURLcode Curl_ftp_nextconnect(struct connectdata *conn)
E:
125 Data connection already open; Transfer starting. */
int size=-1; /* default unknown size */
off_t size=-1; /* default unknown size */
/*
@ -2018,7 +2028,7 @@ CURLcode Curl_ftp_nextconnect(struct connectdata *conn)
/* only if we have nothing but digits: */
if(bytes++) {
/* get the number! */
size = atoi(bytes);
size = strtoofft(bytes, NULL, 0);
}
}
@ -2041,10 +2051,10 @@ CURLcode Curl_ftp_nextconnect(struct connectdata *conn)
return result;
}
if(size > conn->maxdownload)
if(size > conn->maxdownload && conn->maxdownload > 0)
size = conn->size = conn->maxdownload;
infof(data, "Getting file with size: %d\n", size);
infof(data, "Getting file with size: %Od\n", size);
/* FTP download: */
result=Curl_Transfer(conn, SECONDARYSOCKET, size, FALSE,
@ -2160,7 +2170,7 @@ CURLcode ftp_perform(struct connectdata *conn,
/* The SIZE command is _not_ RFC 959 specified, and therefor many servers
may not support it! It is however the only way we have to get a file's
size! */
ssize_t filesize;
off_t filesize;
ssize_t nread;
int ftpcode;
@ -2176,7 +2186,7 @@ CURLcode ftp_perform(struct connectdata *conn,
result = ftp_getsize(conn, ftp->file, &filesize);
if(CURLE_OK == result) {
sprintf(buf, "Content-Length: %d\r\n", filesize);
sprintf(buf, "Content-Length: %Od\r\n", filesize);
result = Curl_client_write(data, CLIENTWRITE_BOTH, buf, 0);
if(result)
return result;

30
lib/http.c
View File

@ -1227,25 +1227,25 @@ CURLcode Curl_http(struct connectdata *conn)
if(conn->resume_from) {
/* do we still game? */
int passed=0;
off_t passed=0;
/* Now, let's read off the proper amount of bytes from the
input. If we knew it was a proper file we could've just
fseek()ed but we only have a stream here */
do {
int readthisamountnow = (conn->resume_from - passed);
int actuallyread;
off_t readthisamountnow = (conn->resume_from - passed);
off_t actuallyread;
if(readthisamountnow > BUFSIZE)
readthisamountnow = BUFSIZE;
actuallyread =
data->set.fread(data->state.buffer, 1, readthisamountnow,
data->set.fread(data->state.buffer, 1, (size_t)readthisamountnow,
data->set.in);
passed += actuallyread;
if(actuallyread != readthisamountnow) {
failf(data, "Could only read %d bytes from the input",
failf(data, "Could only read %Od bytes from the input",
passed);
return CURLE_READ_ERROR;
}
@ -1281,16 +1281,18 @@ CURLcode Curl_http(struct connectdata *conn)
if(conn->resume_from) {
/* This is because "resume" was selected */
long total_expected_size= conn->resume_from + data->set.infilesize;
conn->allocptr.rangeline = aprintf("Content-Range: bytes %s%ld/%ld\r\n",
conn->range, total_expected_size-1,
total_expected_size);
off_t total_expected_size= conn->resume_from + data->set.infilesize;
conn->allocptr.rangeline =
aprintf("Content-Range: bytes %s%Od/%Od\r\n",
conn->range, total_expected_size-1,
total_expected_size);
}
else {
/* Range was selected and then we just pass the incoming range and
append total size */
conn->allocptr.rangeline = aprintf("Content-Range: bytes %s/%d\r\n",
conn->range, data->set.infilesize);
conn->allocptr.rangeline =
aprintf("Content-Range: bytes %s/%Od\r\n",
conn->range, data->set.infilesize);
}
}
}
@ -1519,13 +1521,13 @@ CURLcode Curl_http(struct connectdata *conn)
if((data->set.infilesize>0) && !conn->bits.upload_chunky)
/* only add Content-Length if not uploading chunked */
add_bufferf(req_buffer,
"Content-Length: %d\r\n", /* file size */
"Content-Length: %Od\r\n", /* file size */
data->set.infilesize );
add_bufferf(req_buffer, "\r\n");
/* set the upload size to the progress meter */
Curl_pgrsSetUploadSize(data, data->set.infilesize);
Curl_pgrsSetUploadSize(data, (double)data->set.infilesize);
/* this sends the buffer and frees all the buffer resources */
result = add_buffer_send(req_buffer, conn,
@ -1602,7 +1604,7 @@ CURLcode Curl_http(struct connectdata *conn)
}
else {
/* set the upload size to the progress meter */
Curl_pgrsSetUploadSize(data, data->set.infilesize);
Curl_pgrsSetUploadSize(data, (double)data->set.infilesize);
/* set the pointer to mark that we will send the post body using
the read callback */

9
lib/mprintf.c
View File

@ -167,7 +167,7 @@ static BOOL dprintf_IsQualifierNoDollar(char c)
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
case 'h': case 'l': case 'L': case 'Z': case 'q':
case '*':
case '*': case 'O':
return TRUE;
default:
return FALSE;
@ -376,6 +376,13 @@ static int dprintf_Pass1(char *format, va_stack_t *vto, char **endpos, va_list a
if (sizeof(size_t) > sizeof(unsigned int))
flags |= FLAGS_LONG;
break;
case 'O':
if (sizeof(off_t) > sizeof(unsigned long int)) {
flags |= FLAGS_LONGLONG;
} else if (sizeof(off_t) > sizeof(unsigned int)) {
flags |= FLAGS_LONG;
}
break;
case '0':
if (!(flags & FLAGS_LEFT))
flags |= FLAGS_PAD_NIL;

3
lib/progress.c
View File

@ -228,7 +228,8 @@ int Curl_pgrsUpdate(struct connectdata *conn)
else if(!(data->progress.flags & PGRS_HEADERS_OUT)) {
if (!data->progress.callback) {
if(conn->resume_from)
fprintf(data->set.err, "** Resuming transfer from byte position %d\n",
fprintf(data->set.err,
"** Resuming transfer from byte position %Od\n",
conn->resume_from);
fprintf(data->set.err,
" %% Total %% Received %% Xferd Average Speed Time Curr.\n"

50
lib/transfer.c
View File

@ -34,6 +34,7 @@
#include <errno.h>
#include "strtoofft.h"
#include "strequal.h"
#if defined(WIN32) && !defined(__GNUC__) || defined(__MINGW32__)
@ -599,8 +600,8 @@ CURLcode Curl_readwrite(struct connectdata *conn,
resuming a document that we don't get, and this header contains
info about the true size of the document we didn't get now. */
if ((k->httpcode != 416) &&
checkprefix("Content-Length:", k->p) &&
sscanf (k->p+15, " %ld", &contentlength)) {
checkprefix("Content-Length:", k->p)) {
contentlength = strtoofft(k->p+15, NULL, 10);
if (data->set.max_filesize && contentlength >
data->set.max_filesize) {
failf(data, "Maximum file size exceeded");
@ -718,17 +719,26 @@ CURLcode Curl_readwrite(struct connectdata *conn,
|| checkprefix("x-compress", start))
k->content_encoding = COMPRESS;
}
else if (checkprefix("Content-Range:", k->p)) {
if (sscanf (k->p+14, " bytes %d-", &k->offset) ||
sscanf (k->p+14, " bytes: %d-", &k->offset)) {
/* This second format was added August 1st 2000 by Igor
Khristophorov since Sun's webserver JavaWebServer/1.1.1
obviously sends the header this way! :-( */
if (conn->resume_from == k->offset) {
/* we asked for a resume and we got it */
k->content_range = TRUE;
}
}
else if (Curl_compareheader(k->p, "Content-Range:", "bytes")) {
/* Content-Range: bytes [num]-
Content-Range: bytes: [num]-
The second format was added August 1st 2000 by Igor
Khristophorov since Sun's webserver JavaWebServer/1.1.1
obviously sends the header this way! :-( */
char *ptr = strstr(k->p, "bytes");
ptr+=5;
if(*ptr == ':')
/* stupid colon skip */
ptr++;
k->offset = strtoofft(ptr, NULL, 10);
if (conn->resume_from == k->offset)
/* we asked for a resume and we got it */
k->content_range = TRUE;
}
else if(data->cookies &&
checkprefix("Set-Cookie:", k->p)) {
@ -947,7 +957,7 @@ CURLcode Curl_readwrite(struct connectdata *conn,
if((-1 != conn->maxdownload) &&
(k->bytecount + nread >= conn->maxdownload)) {
nread = conn->maxdownload - k->bytecount;
nread = (ssize_t) (conn->maxdownload - k->bytecount);
if(nread < 0 ) /* this should be unusual */
nread = 0;
@ -1213,7 +1223,7 @@ CURLcode Curl_readwrite(struct connectdata *conn,
if (data->set.timeout &&
((Curl_tvdiff(k->now, k->start)/1000) >= data->set.timeout)) {
failf (data, "Operation timed out with %d out of %d bytes received",
failf (data, "Operation timed out with %Od out of %Od bytes received",
k->bytecount, conn->size);
return CURLE_OPERATION_TIMEOUTED;
}
@ -1227,7 +1237,7 @@ CURLcode Curl_readwrite(struct connectdata *conn,
if(!(data->set.no_body) && (conn->size != -1) &&
(k->bytecount != conn->size) &&
!conn->newurl) {
failf(data, "transfer closed with %d bytes remaining to read",
failf(data, "transfer closed with %Od bytes remaining to read",
conn->size - k->bytecount);
return CURLE_PARTIAL_FILE;
}
@ -1277,7 +1287,7 @@ CURLcode Curl_readwrite_init(struct connectdata *conn)
if (!conn->bits.getheader) {
k->header = FALSE;
if(conn->size > 0)
Curl_pgrsSetDownloadSize(data, conn->size);
Curl_pgrsSetDownloadSize(data, (double)conn->size);
}
/* we want header and/or body, if neither then don't do this! */
if(conn->bits.getheader || !data->set.no_body) {
@ -1946,13 +1956,13 @@ CURLcode Curl_perform(struct SessionHandle *data)
CURLcode
Curl_Transfer(struct connectdata *c_conn, /* connection data */
int sockindex, /* socket index to read from or -1 */
int size, /* -1 if unknown at this point */
off_t size, /* -1 if unknown at this point */
bool getheader, /* TRUE if header parsing is wanted */
long *bytecountp, /* return number of bytes read or NULL */
off_t *bytecountp, /* return number of bytes read or NULL */
int writesockindex, /* socket index to write to, it may very
well be the same we read from. -1
disables */
long *writebytecountp /* return number of bytes written or
off_t *writebytecountp /* return number of bytes written or
NULL */
)
{

6
lib/transfer.h
View File

@ -38,11 +38,11 @@ CURLcode Curl_readwrite_init(struct connectdata *conn);
CURLcode
Curl_Transfer (struct connectdata *data,
int sockfd, /* socket to read from or -1 */
int size, /* -1 if unknown at this point */
off_t size, /* -1 if unknown at this point */
bool getheader, /* TRUE if header parsing is wanted */
long *bytecountp, /* return number of bytes read */
off_t *bytecountp, /* return number of bytes read */
int writesockfd, /* socket to write to, it may very well be
the same we read from. -1 disables */
long *writebytecountp /* return number of bytes written */
off_t *writebytecountp /* return number of bytes written */
);
#endif

26
lib/url.c
View File

@ -711,6 +711,13 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option, ...)
*/
data->set.infilesize = va_arg(param, long);
break;
case CURLOPT_INFILESIZE_LARGE:
/*
* If known, this should inform curl about the file size of the
* to-be-uploaded file.
*/
data->set.infilesize = va_arg(param, off_t);
break;
case CURLOPT_LOW_SPEED_LIMIT:
/*
* The low speed limit that if transfers are below this for
@ -955,6 +962,12 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option, ...)
*/
data->set.set_resume_from = va_arg(param, long);
break;
case CURLOPT_RESUME_FROM_LARGE:
/*
* Resume transfer at the give file position
*/
data->set.set_resume_from = va_arg(param, off_t);
break;
case CURLOPT_DEBUGFUNCTION:
/*
* stderr write callback.
@ -1262,6 +1275,13 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option, ...)
data->set.ip_version = va_arg(param, long);
break;
case CURLOPT_MAXFILESIZE_LARGE:
/*
* Set the maximum size of a file to download.
*/
data->set.max_filesize = va_arg(param, off_t);
break;
default:
/* unknown tag and its companion, just ignore: */
return CURLE_FAILED_INIT; /* correct this */
@ -2348,7 +2368,8 @@ static CURLcode CreateConnection(struct SessionHandle *data,
if(conn->resume_from) {
if(!conn->bits.use_range) {
/* if it already was in use, we just skip this */
snprintf(resumerange, sizeof(resumerange), "%d-", conn->resume_from);
snprintf(resumerange, sizeof(resumerange), "%Od-",
conn->resume_from);
conn->range=strdup(resumerange); /* tell ourselves to fetch this range */
conn->bits.rangestringalloc = TRUE; /* mark as allocated */
conn->bits.use_range = 1; /* switch on range usage */
@ -2869,7 +2890,8 @@ static CURLcode CreateConnection(struct SessionHandle *data,
*/
conn->resume_from = data->set.set_resume_from;
if (conn->resume_from) {
snprintf(resumerange, sizeof(resumerange), "%d-", conn->resume_from);
snprintf(resumerange, sizeof(resumerange), "%Od-",
conn->resume_from);
if (conn->bits.rangestringalloc == TRUE)
free(conn->range);

32
lib/urldata.h
View File

@ -210,8 +210,8 @@ struct HTTP {
const char *p_pragma; /* Pragma: string */
const char *p_accept; /* Accept: string */
long readbytecount;
long writebytecount;
off_t readbytecount;
off_t writebytecount;
/* For FORM posting */
struct Form form;
@ -239,7 +239,7 @@ struct HTTP {
* FTP unique setup
***************************************************************************/
struct FTP {
long *bytecountp;
off_t *bytecountp;
char *user; /* user name string */
char *passwd; /* password string */
char *urlpath; /* the originally given path part of the URL */
@ -249,7 +249,7 @@ struct FTP {
char *entrypath; /* the PWD reply when we logged on */
char *cache; /* data cache between getresponse()-calls */
size_t cache_size; /* size of cache in bytes */
off_t cache_size; /* size of cache in bytes */
bool dont_check; /* Set to TRUE to prevent the final (post-transfer)
file size and 226/250 status check. It should still
read the line, just ignore the result. */
@ -304,7 +304,7 @@ struct ConnectBits {
*/
struct Curl_transfer_keeper {
int bytecount; /* total number of bytes read */
off_t bytecount; /* total number of bytes read */
int writebytecount; /* number of bytes written */
struct timeval start; /* transfer started at this time */
struct timeval now; /* current time */
@ -325,7 +325,7 @@ struct Curl_transfer_keeper {
char *end_ptr; /* within buf */
char *p; /* within headerbuff */
bool content_range; /* set TRUE if Content-Range: was found */
int offset; /* possible resume offset read from the
off_t offset; /* possible resume offset read from the
Content-Range: header */
int httpcode; /* error code from the 'HTTP/1.? XXX' line */
int httpversion; /* the HTTP version*10 */
@ -427,12 +427,12 @@ struct connectdata {
unsigned short remote_port; /* what remote port to connect to,
not the proxy port! */
char *ppath;
long bytecount;
off_t bytecount;
long headerbytecount; /* only count received headers */
char *range; /* range, if used. See README for detailed specification on
this syntax. */
ssize_t resume_from; /* continue [ftp] transfer from here */
off_t resume_from; /* continue [ftp] transfer from here */
char *proxyhost; /* name of the http proxy host */
@ -446,8 +446,8 @@ struct connectdata {
struct timeval created; /* creation time */
int sock[2]; /* two sockets, the second is used for the data transfer
when doing FTP */
long maxdownload; /* in bytes, the maximum amount of data to fetch, 0
means unlimited */
off_t maxdownload; /* in bytes, the maximum amount of data to fetch, 0
means unlimited */
struct ssl_connect_data ssl[2]; /* this is for ssl-stuff */
struct ssl_config_data ssl_config;
@ -484,13 +484,13 @@ struct connectdata {
/* READ stuff */
int sockfd; /* socket to read from or -1 */
int size; /* -1 if unknown at this point */
long *bytecountp; /* return number of bytes read or NULL */
off_t size; /* -1 if unknown at this point */
off_t *bytecountp; /* return number of bytes read or NULL */
/* WRITE stuff */
int writesockfd; /* socket to write to, it may very
well be the same we read from. -1 disables */
long *writebytecountp; /* return number of bytes written or NULL */
off_t *writebytecountp; /* return number of bytes written or NULL */
/** Dynamicly allocated strings, may need to be freed before this **/
/** struct is killed. **/
@ -784,10 +784,10 @@ struct UserDefined {
long timeout; /* in seconds, 0 means no timeout */
long connecttimeout; /* in seconds, 0 means no timeout */
long ftp_response_timeout; /* in seconds, 0 means no timeout */
long infilesize; /* size of file to upload, -1 means unknown */
off_t infilesize; /* size of file to upload, -1 means unknown */
long low_speed_limit; /* bytes/second */
long low_speed_time; /* number of seconds */
int set_resume_from; /* continue [ftp] transfer from here */
off_t set_resume_from; /* continue [ftp] transfer from here */
char *cookie; /* HTTP cookie string to send */
struct curl_slist *headers; /* linked list of extra headers */
struct HttpPost *httppost; /* linked list of POST data */
@ -829,7 +829,7 @@ struct UserDefined {
int ip_version;
long max_filesize; /* Maximum file size to download */
off_t max_filesize; /* Maximum file size to download */
/* Here follows boolean settings that define how to behave during
this session. They are STATIC, set by libcurl users or at least initially

6
src/config.h.in
View File

@ -76,3 +76,9 @@
/* Define to 1 if you have the <pwd.h> header file. */
#undef HAVE_PWD_H
/* Define to 1 if you have the `strtoll' function. */
#undef HAVE_STRTOLL
/* The size of a `off_t', as computed by sizeof. */
#undef SIZEOF_OFF_T

55
src/main.c
View File

@ -388,14 +388,14 @@ struct Configurable {
bool resume_from_current;
bool disable_epsv;
bool disable_eprt;
long resume_from;
off_t resume_from;
char *postfields;
long postfieldsize;
char *referer;
long timeout;
long connecttimeout;
long maxredirs;
long max_filesize;
off_t max_filesize;
char *headerfile;
char *ftpport;
char *iface;
@ -984,6 +984,42 @@ static int str2num(long *val, char *str)
return retcode;
}
/**
* Parses the given string looking for an offset (which may be
* a larger-than-integer value).
*
* @param val the offset to populate
* @param str the buffer containing the offset
* @return zero if successful, non-zero if failure.
*/
static int str2offset(off_t *val, char *str)
{
#if SIZEOF_OFF_T > 4
/* Ugly, but without going through a bunch of rigamarole,
* we don't have the definitions for LLONG_{MIN,MAX} or
* LONG_LONG_{MIN,MAX}.
*/
#ifndef LLONG_MAX
#define LLONG_MAX (off_t)0x7FFFFFFFFFFFFFFFLL
#define LLONG_MIN (off_t)0x8000000000000000LL
#endif
#ifdef HAVE_STRTOLL
*val = strtoll(str, NULL, 0);
#else
/* TODO: Handle strtoll stuff...sigh... */
#endif
if ((*val == LLONG_MAX || *val == LLONG_MIN) && errno == ERANGE)
return 1;
#else
*val = strtol(str, NULL, 0);
if ((*val == LONG_MIN || *val == LONG_MAX) && errno == ERANGE)
return 1;
#endif
return 0;
}
static void checkpasswd(const char *kind, /* for what purpose */
char **userpwd) /* pointer to allocated string */
{
@ -1353,7 +1389,7 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */
return PARAM_LIBCURL_DOESNT_SUPPORT;
break;
case 'y': /* --max-filesize */
if(str2num(&config->max_filesize, nextarg))
if(str2offset(&config->max_filesize, nextarg))
return PARAM_BAD_NUMERIC;
break;
case 'z': /* --disable-eprt */
@ -1457,7 +1493,7 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */
case 'C':
/* This makes us continue an ftp transfer at given position */
if(!curl_strequal(nextarg, "-")) {
if(str2num(&config->resume_from, nextarg))
if(str2offset(&config->resume_from, nextarg))
return PARAM_BAD_NUMERIC;
config->resume_from_current = FALSE;
}
@ -2221,7 +2257,7 @@ struct ProgressData {
double prev;
int width;
FILE *out; /* where to write everything to */
int initial_size;
off_t initial_size;
};
int myprogress (void *clientp,
@ -2523,7 +2559,7 @@ operate(struct Configurable *config, int argc, char *argv[])
bool infdfopen;
FILE *headerfilep = NULL;
char *urlbuffer=NULL;
long uploadfilesize; /* -1 means unknown */
off_t uploadfilesize; /* -1 means unknown */
bool stillflags=TRUE;
bool allocuseragent=FALSE;
@ -3029,7 +3065,7 @@ operate(struct Configurable *config, int argc, char *argv[])
}
/* size of uploaded file: */
curl_easy_setopt(curl, CURLOPT_INFILESIZE, uploadfilesize);
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, uploadfilesize);
curl_easy_setopt(curl, CURLOPT_URL, url); /* what to fetch */
curl_easy_setopt(curl, CURLOPT_PROXY, config->proxy); /* proxy to use */
curl_easy_setopt(curl, CURLOPT_HEADER, config->conf&CONF_HEADER);
@ -3072,7 +3108,7 @@ operate(struct Configurable *config, int argc, char *argv[])
curl_easy_setopt(curl, CURLOPT_FTPPORT, config->ftpport);
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, config->low_speed_limit);
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, config->low_speed_time);
curl_easy_setopt(curl, CURLOPT_RESUME_FROM,
curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE,
config->use_resume?config->resume_from:0);
curl_easy_setopt(curl, CURLOPT_COOKIE, config->cookie);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, config->headers);
@ -3190,7 +3226,8 @@ operate(struct Configurable *config, int argc, char *argv[])
/* new in curl 7.10.8 */
if(config->max_filesize)
curl_easy_setopt(curl, CURLOPT_MAXFILESIZE, config->max_filesize);
curl_easy_setopt(curl, CURLOPT_MAXFILESIZE_LARGE,
config->max_filesize);
/* new in curl 7.10.9 */
if(config->ftp_ssl)

4
tests/libtest/lib505.c
View File

@ -92,8 +92,8 @@ CURLcode test(char *URL)
curl_easy_setopt(curl, CURLOPT_INFILE, hd_src);
/* and give the size of the upload (optional) */
curl_easy_setopt(curl, CURLOPT_INFILESIZE,
(long)file_info.st_size);
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE,
file_info.st_size);
/* Now run off and do what you've been told! */
res = curl_easy_perform(curl);