generic-library/curl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Multiple URL support added

Browse Source
This commit is contained in:
Daniel Stenberg
2001-01-08 07:37:44 +00:00
parent 53c27c7722
commit 14ca732a8f
3 changed files with 396 additions and 312 deletions
Download Patch File Download Diff File Expand all files Collapse all files

245
src/main.c
View File

@@ -39,7 +39,6 @@
#include "writeout.h" #include "writeout.h"
#define CURLseparator "--_curl_--" #define CURLseparator "--_curl_--"
#define MIMEseparator "_curl_"
/* This define make use of the "Curlseparator" as opposed to the /* This define make use of the "Curlseparator" as opposed to the
MIMEseparator. We might add support for the latter one in the MIMEseparator. We might add support for the latter one in the
@@ -222,6 +221,20 @@ static void helpf(char *fmt, ...)
fprintf(stderr, "curl: try 'curl --help' for more information\n"); fprintf(stderr, "curl: try 'curl --help' for more information\n");
} }
/*
* A chain of these nodes contain URL to get and where to put the URL's
* contents.
*/
struct getout {
struct getout *next;
char *url;
char *outfile;
int flags;
};
#define GETOUT_OUTFILE (1<<0) /* set when outfile is deemed done */
#define GETOUT_URL (1<<1) /* set when URL is deemed done */
#define GETOUT_USEREMOTE (1<<2) /* use remote file name locally */
static void help(void) static void help(void)
{ {
printf(CURL_ID "%s\n" printf(CURL_ID "%s\n"
@@ -303,9 +316,7 @@ struct Configurable {
char *referer; char *referer;
long timeout; long timeout;
long maxredirs; long maxredirs;
char *outfile;
char *headerfile; char *headerfile;
char remotefile;
char *ftpport; char *ftpport;
char *iface; char *iface;
unsigned short porttouse; unsigned short porttouse;
@@ -320,7 +331,13 @@ struct Configurable {
bool configread; bool configread;
bool proxytunnel; bool proxytunnel;
long conf; long conf;
char *url;
struct getout *url_list; /* point to the first node */
struct getout *url_last; /* point to the last/current node */
struct getout *url_get; /* point to the node to fill in URL */
struct getout *url_out; /* point to the node to fill in outfile */
char *cert; char *cert;
char *cacert; char *cacert;
char *cert_passwd; char *cert_passwd;
@@ -426,6 +443,29 @@ static char *file2memory(FILE *file, long *size)
return NULL; /* no string */ return NULL; /* no string */
} }
struct getout *new_getout(struct Configurable *config)
{
struct getout *node =malloc(sizeof(struct getout));
struct getout *last= config->url_last;
if(node) {
/* clear the struct */
memset(node, 0, sizeof(struct getout));
/* append this new node last in the list */
if(last)
last->next = node;
else
config->url_list = node; /* first node */
/* move the last pointer */
config->url_last = node;
}
return node;
}
typedef enum { typedef enum {
PARAM_OK, PARAM_OK,
PARAM_OPTION_AMBIGUOUS, PARAM_OPTION_AMBIGUOUS,
@@ -610,7 +650,30 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */
break; break;
case '5': case '5':
/* the URL! */ /* the URL! */
GetStr(&config->url, nextarg); {
struct getout *url;
if(config->url_get || (config->url_get=config->url_list)) {
/* there's a node here, if it already is filled-in continue to find
an "empty" node */
while(config->url_get && (config->url_get->flags&GETOUT_URL))
config->url_get = config->url_get->next;
}
/* now there might or might not be an available node to fill in! */
if(config->url_get)
/* existing node */
url = config->url_get;
else
/* there was no free node, create one! */
url=new_getout(config);
if(url) {
/* fill in the URL */
GetStr(&url->url, nextarg);
url->flags |= GETOUT_URL;
}
}
break; break;
case '#': /* added 19990617 larsa */ case '#': /* added 19990617 larsa */
config->progressmode ^= CURL_PROGRESS_BAR; config->progressmode ^= CURL_PROGRESS_BAR;
@@ -794,12 +857,37 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */
config->nobuffer ^= 1; config->nobuffer ^= 1;
break; break;
case 'o': case 'o':
/* output file */
GetStr(&config->outfile, nextarg); /* write to this file */
break;
case 'O': case 'O':
/* output file */ /* output file */
config->remotefile ^= TRUE; {
struct getout *url;
if(config->url_out || (config->url_out=config->url_list)) {
/* there's a node here, if it already is filled-in continue to find
an "empty" node */
while(config->url_out && (config->url_out->flags&GETOUT_OUTFILE))
config->url_out = config->url_out->next;
}
/* now there might or might not be an available node to fill in! */
if(config->url_out)
/* existing node */
url = config->url_out;
else
/* there was no free node, create one! */
url=new_getout(config);
if(url) {
/* fill in the outfile */
if('o' == letter)
GetStr(&url->outfile, nextarg);
else {
url->outfile=NULL; /* leave it */
url->flags |= GETOUT_USEREMOTE;
}
url->flags |= GETOUT_OUTFILE;
}
}
break; break;
case 'P': case 'P':
/* This makes the FTP sessions use PORT instead of PASV */ /* This makes the FTP sessions use PORT instead of PASV */
@@ -1253,8 +1341,6 @@ void progressbarinit(struct ProgressData *bar)
void free_config_fields(struct Configurable *config) void free_config_fields(struct Configurable *config)
{ {
if(config->url)
free(config->url);
if(config->userpwd) if(config->userpwd)
free(config->userpwd); free(config->userpwd);
if(config->postfields) if(config->postfields)
@@ -1271,8 +1357,6 @@ void free_config_fields(struct Configurable *config)
free(config->krb4level); free(config->krb4level);
if(config->headerfile) if(config->headerfile)
free(config->headerfile); free(config->headerfile);
if(config->outfile)
free(config->outfile);
if(config->ftpport) if(config->ftpport)
free(config->ftpport); free(config->ftpport);
if(config->infile) if(config->infile)
@@ -1300,6 +1384,8 @@ operate(struct Configurable *config, int argc, char *argv[])
char errorbuffer[CURL_ERROR_SIZE]; char errorbuffer[CURL_ERROR_SIZE];
char useragent[128]; /* buah, we don't want a larger default user agent */ char useragent[128]; /* buah, we don't want a larger default user agent */
struct ProgressData progressbar; struct ProgressData progressbar;
struct getout *urlnode;
struct getout *nextnode;
struct OutStruct outs; struct OutStruct outs;
struct OutStruct heads; struct OutStruct heads;
@@ -1323,9 +1409,6 @@ operate(struct Configurable *config, int argc, char *argv[])
int res; int res;
int i; int i;
outs.stream = stdout;
outs.config = config;
#ifdef MALLOCDEBUG #ifdef MALLOCDEBUG
/* this sends all memory debug messages to a logfile named memdump */ /* this sends all memory debug messages to a logfile named memdump */
curl_memdebug("memdump"); curl_memdebug("memdump");
@@ -1356,7 +1439,7 @@ operate(struct Configurable *config, int argc, char *argv[])
return res; return res;
} }
if ((argc < 2) && !config->url) { if ((argc < 2) && !config->url_list) {
helpf(NULL); helpf(NULL);
return CURLE_FAILED_INIT; return CURLE_FAILED_INIT;
} }
@@ -1405,20 +1488,15 @@ operate(struct Configurable *config, int argc, char *argv[])
} }
} }
else { else {
if(url) { bool used;
helpf("only one URL is supported!\n"); /* just add the URL please */
return CURLE_FAILED_INIT; res = getparameter("--url", argv[i], &used, config);
} if(res)
url = argv[i]; return res;
} }
} }
/* if no URL was specified and there was one in the config file, get that if(!config->url_list) {
one */
if(!url && config->url)
url = config->url;
if(!url) {
helpf("no URL specified!\n"); helpf("no URL specified!\n");
return CURLE_FAILED_INIT; return CURLE_FAILED_INIT;
} }
@@ -1430,9 +1508,18 @@ operate(struct Configurable *config, int argc, char *argv[])
} }
else else
allocuseragent = TRUE; allocuseragent = TRUE;
#if 0
fprintf(stderr, "URL: %s PROXY: %s\n", url, config->proxy?config->proxy:"none"); urlnode = config->url_list;
#endif
/* loop through the list of given URLs */
while(urlnode) {
/* get the full URL */
url=urlnode->url;
/* default output stream is stdout */
outs.stream = stdout;
outs.config = config;
/* expand '{...}' and '[...]' expressions and return total number of URLs /* expand '{...}' and '[...]' expressions and return total number of URLs
in pattern set */ in pattern set */
@@ -1440,34 +1527,27 @@ operate(struct Configurable *config, int argc, char *argv[])
if(res != CURLE_OK) if(res != CURLE_OK)
return res; return res;
/* save outfile pattern befor expansion */ /* save outfile pattern before expansion */
outfiles = config->outfile?strdup(config->outfile):NULL; outfiles = urlnode->outfile?strdup(urlnode->outfile):NULL;
if (!outfiles && !config->remotefile && urlnum > 1) { if (outfiles && strequal(outfiles, "-") && urlnum > 1) {
#ifdef CURL_SEPARATORS
/* multiple files extracted to stdout, insert separators! */ /* multiple files extracted to stdout, insert separators! */
separator = 1; separator = 1;
#endif
#ifdef MIME_SEPARATORS
/* multiple files extracted to stdout, insert MIME separators! */
separator = 1;
printf("MIME-Version: 1.0\n");
printf("Content-Type: multipart/mixed; boundary=%s\n\n", MIMEseparator);
#endif
} }
for (i = 0; (url = next_url(urls)); ++i) { for (i = 0; (url = next_url(urls)); ++i) {
if (config->outfile) { char *outfile;
free(config->outfile); outfile = outfiles?strdup(outfiles):NULL;
config->outfile = outfiles?strdup(outfiles):NULL;
}
if((urlnode->flags&GETOUT_USEREMOTE) ||
(outfile && !strequal("-", outfile)) ) {
if (config->outfile || config->remotefile) {
/* /*
* We have specified a file name to store the result in, or we have * We have specified a file name to store the result in, or we have
* decided we want to use the remote file name. * decided we want to use the remote file name.
*/ */
if(!config->outfile && config->remotefile) { if(!outfile) {
/* Find and get the remote file name */ /* Find and get the remote file name */
char * pc =strstr(url, "://"); char * pc =strstr(url, "://");
if(pc) if(pc)
@@ -1475,25 +1555,26 @@ operate(struct Configurable *config, int argc, char *argv[])
else else
pc=url; pc=url;
pc = strrchr(pc, '/'); pc = strrchr(pc, '/');
config->outfile = (char *) NULL == pc ? NULL : strdup(pc+1) ; outfile = (char *) NULL == pc ? NULL : strdup(pc+1) ;
if(!config->outfile || !strlen(config->outfile)) { if(!outfile) {
helpf("Remote file name has no length!\n"); helpf("Remote file name has no length!\n");
return CURLE_WRITE_ERROR; return CURLE_WRITE_ERROR;
} }
} }
else { else {
/* fill '#1' ... '#9' terms from URL pattern */ /* fill '#1' ... '#9' terms from URL pattern */
char *outfile = config->outfile; char *storefile = outfile;
config->outfile = match_url(config->outfile, urls); outfile = match_url(storefile, urls);
free(outfile); free(storefile);
} }
if((0 == config->resume_from) && config->use_resume) { if((0 == config->resume_from) && config->use_resume) {
/* we're told to continue where we are now, then we get the size of the /* we're told to continue where we are now, then we get the size of
file as it is now and open it for append instead */ the file as it is now and open it for append instead */
struct stat fileinfo; struct stat fileinfo;
if(0 == stat(config->outfile, &fileinfo)) { if(0 == stat(outfile, &fileinfo)) {
/* set offset to current file size: */ /* set offset to current file size: */
config->resume_from = fileinfo.st_size; config->resume_from = fileinfo.st_size;
} }
@@ -1502,14 +1583,14 @@ operate(struct Configurable *config, int argc, char *argv[])
if(config->resume_from) { if(config->resume_from) {
/* open file for output: */ /* open file for output: */
outs.stream=(FILE *) fopen(config->outfile, config->resume_from?"ab":"wb"); outs.stream=(FILE *) fopen(outfile, config->resume_from?"ab":"wb");
if (!outs.stream) { if (!outs.stream) {
helpf("Can't open '%s'!\n", config->outfile); helpf("Can't open '%s'!\n", outfile);
return CURLE_WRITE_ERROR; return CURLE_WRITE_ERROR;
} }
} }
else { else {
outs.filename = config->outfile; outs.filename = outfile;
outs.stream = NULL; /* open when needed */ outs.stream = NULL; /* open when needed */
} }
} }
@@ -1527,8 +1608,9 @@ operate(struct Configurable *config, int argc, char *argv[])
ptr=url; ptr=url;
ptr = strrchr(ptr, '/'); ptr = strrchr(ptr, '/');
if(!ptr || !strlen(++ptr)) { if(!ptr || !strlen(++ptr)) {
/* The URL has no file name part, add the local file name. In order to /* The URL has no file name part, add the local file name. In order
be able to do so, we have to create a new URL in another buffer.*/ to be able to do so, we have to create a new URL in another
buffer.*/
urlbuffer=(char *)malloc(strlen(url) + strlen(config->infile) + 3); urlbuffer=(char *)malloc(strlen(url) + strlen(config->infile) + 3);
if(!urlbuffer) { if(!urlbuffer) {
@@ -1579,23 +1661,16 @@ operate(struct Configurable *config, int argc, char *argv[])
if (urlnum > 1) { if (urlnum > 1) {
fprintf(stderr, "\n[%d/%d]: %s --> %s\n", fprintf(stderr, "\n[%d/%d]: %s --> %s\n",
i+1, urlnum, url, config->outfile ? config->outfile : "<stdout>"); i+1, urlnum, url, outfile ? outfile : "<stdout>");
if (separator) { if (separator)
#ifdef CURL_SEPARATORS
printf("%s%s\n", CURLseparator, url); printf("%s%s\n", CURLseparator, url);
#endif
#ifdef MIME_SEPARATORS
printf("--%s\n", MIMEseparator);
printf("Content-ID: %s\n\n", url);
#endif
}
} }
if(!config->errors) if(!config->errors)
config->errors = stderr; config->errors = stderr;
#ifdef WIN32 #ifdef WIN32
if(!config->outfile && !(config->conf & CONF_GETTEXT)) { if(!outfile && !(config->conf & CONF_GETTEXT)) {
/* We get the output to stdout and we have not got the ASCII/text flag, /* We get the output to stdout and we have not got the ASCII/text flag,
then set stdout to be binary */ then set stdout to be binary */
setmode( 1, O_BINARY ); setmode( 1, O_BINARY );
@@ -1605,7 +1680,6 @@ operate(struct Configurable *config, int argc, char *argv[])
main_init(); main_init();
/* The new, v7-style easy-interface! */
curl = curl_easy_init(); curl = curl_easy_init();
if(curl) { if(curl) {
curl_easy_setopt(curl, CURLOPT_FILE, (FILE *)&outs); /* where to store */ curl_easy_setopt(curl, CURLOPT_FILE, (FILE *)&outs); /* where to store */
@@ -1676,7 +1750,6 @@ operate(struct Configurable *config, int argc, char *argv[])
else else
curl_easy_setopt(curl, CURLOPT_MAXREDIRS, DEFAULT_MAXREDIRS); curl_easy_setopt(curl, CURLOPT_MAXREDIRS, DEFAULT_MAXREDIRS);
curl_easy_setopt(curl, CURLOPT_CRLF, config->crlf); curl_easy_setopt(curl, CURLOPT_CRLF, config->crlf);
curl_easy_setopt(curl, CURLOPT_QUOTE, config->quote); curl_easy_setopt(curl, CURLOPT_QUOTE, config->quote);
curl_easy_setopt(curl, CURLOPT_POSTQUOTE, config->postquote); curl_easy_setopt(curl, CURLOPT_POSTQUOTE, config->postquote);
@@ -1730,7 +1803,7 @@ operate(struct Configurable *config, int argc, char *argv[])
if(urlbuffer) if(urlbuffer)
free(urlbuffer); free(urlbuffer);
if (config->outfile && outs.stream) if (outfile && !strequal(outfile, "-") && outs.stream)
fclose(outs.stream); fclose(outs.stream);
if (config->infile) if (config->infile)
fclose(infd); fclose(infd);
@@ -1740,21 +1813,31 @@ operate(struct Configurable *config, int argc, char *argv[])
if(url) if(url)
free(url); free(url);
if(outfile)
free(outfile);
} }
if(outfiles) if(outfiles)
free(outfiles); free(outfiles);
#ifdef MIME_SEPARATORS /* cleanup memory used for URL globbing patterns */
if (separator) glob_cleanup(urls);
printf("--%s--\n", MIMEseparator);
#endif /* empty this urlnode struct */
if(urlnode->url)
free(urlnode->url);
if(urlnode->outfile)
free(urlnode->outfile);
/* move on to the next URL */
nextnode=urlnode->next;
free(urlnode); /* free the node */
urlnode = nextnode;
} /* while-loop through all URLs */
if(allocuseragent) if(allocuseragent)
free(config->useragent); free(config->useragent);
/* cleanup memory used for URL globbing patterns */
glob_cleanup(urls);
return res; return res;
} }

6
src/urlglob.c
View File

@@ -213,6 +213,7 @@ int glob_url(URLGlob** glob, char* url, int *urlnum)
glob_expand->size = 0; glob_expand->size = 0;
glob_expand->urllen = strlen(url); glob_expand->urllen = strlen(url);
glob_expand->glob_buffer = glob_buffer; glob_expand->glob_buffer = glob_buffer;
glob_expand->beenhere=0;
*urlnum = glob_word(glob_expand, url, 1); *urlnum = glob_word(glob_expand, url, 1);
*glob = glob_expand; *glob = glob_expand;
return CURLE_OK; return CURLE_OK;
@@ -240,15 +241,14 @@ void glob_cleanup(URLGlob* glob)
char *next_url(URLGlob *glob) char *next_url(URLGlob *glob)
{ {
static int beenhere = 0;
char *buf = glob->glob_buffer; char *buf = glob->glob_buffer;
URLPattern *pat; URLPattern *pat;
char *lit; char *lit;
signed int i; signed int i;
int carry; int carry;
if (!beenhere) if (!glob->beenhere)
beenhere = 1; glob->beenhere = 1;
else { else {
carry = 1; carry = 1;

1
src/urlglob.h
View File

@@ -50,6 +50,7 @@ typedef struct {
int size; int size;
int urllen; int urllen;
char *glob_buffer; char *glob_buffer;
char beenhere;
} URLGlob; } URLGlob;
int glob_url(URLGlob**, char*, int *); int glob_url(URLGlob**, char*, int *);