generic-library/curl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Introducing a new persistent connection caching system using "bundles".

Browse Source 
A bundle is a list of all persistent connections to the same host.
The connection cache consists of a hash of bundles, with the
hostname as the key.
The benefits may not be obvious, but they are two:

1) Faster search for connections to reuse, since the hash
   lookup only finds connections to the host in question.
2) It lays out the groundworks for an upcoming patch,
   which will introduce multiple HTTP pipelines.

This patch also removes the awkward list of "closure handles",
which were needed to send QUIT commands to the FTP server
when closing a connection.
Now we allocate a separate closure handle and use that
one to close all connections.

This has been tested in a live system for a few weeks, and of
course passes the test suite.
This commit is contained in:
Linus Nielsen Feltzing 2012-12-06 12:12:04 +01:00 committed by Daniel Stenberg
parent ca5f4e2135
commit d021f2e8a0
14 changed files with 894 additions and 773 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lib/Makefile.inc
View File

@ -23,8 +23,9 @@ CSOURCES = file.c timeval.c base64.c hostip.c progress.c formdata.c \
curl_rtmp.c openldap.c curl_gethostname.c gopher.c axtls.c \
idn_win32.c http_negotiate_sspi.c cyassl.c http_proxy.c non-ascii.c \
asyn-ares.c asyn-thread.c curl_gssapi.c curl_ntlm.c curl_ntlm_wb.c \
curl_ntlm_core.c curl_ntlm_msgs.c curl_sasl.c curl_schannel.c \
curl_multibyte.c curl_darwinssl.c hostcheck.c
curl_ntlm_core.c curl_ntlm_msgs.c curl_sasl.c curl_schannel.c \
curl_multibyte.c curl_darwinssl.c hostcheck.c \
bundles.c conncache.c
HHEADERS = arpa_telnet.h netrc.h file.h timeval.h qssl.h hostip.h \
progress.h formdata.h cookie.h http.h sendf.h ftp.h url.h dict.h \
@ -42,4 +43,4 @@ HHEADERS = arpa_telnet.h netrc.h file.h timeval.h qssl.h hostip.h \
gopher.h axtls.h cyassl.h http_proxy.h non-ascii.h asyn.h curl_ntlm.h \
curl_gssapi.h curl_ntlm_wb.h curl_ntlm_core.h curl_ntlm_msgs.h \
curl_sasl.h curl_schannel.h curl_multibyte.h curl_darwinssl.h \
hostcheck.h
hostcheck.h bundles.h conncache.h

101
lib/bundles.c Normal file
View File

@ -0,0 +1,101 @@
/***************************************************************************
* _ _ ____ _
* Project ___| | | | _ \| |
* / __| | | | |_) | |
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 2012, Linus Nielsen Feltzing, <linus@haxx.se>
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
* are also available at http://curl.haxx.se/docs/copyright.html.
*
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
* copies of the Software, and permit persons to whom the Software is
* furnished to do so, under the terms of the COPYING file.
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
* KIND, either express or implied.
*
***************************************************************************/
#include "setup.h"
#include <curl/curl.h>
#include "urldata.h"
#include "url.h"
#include "progress.h"
#include "multiif.h"
#include "bundles.h"
#include "sendf.h"
#include "rawstr.h"
#include "curl_memory.h"
/* The last #include file should be: */
#include "memdebug.h"
static void conn_llist_dtor(void *user, void *element)
{
struct connectdata *data = element;
(void)user;
data->bundle = NULL;
}
CURLcode Curl_bundle_create(struct SessionHandle *data,
struct connectbundle **cb_ptr)
{
(void)data;
*cb_ptr = malloc(sizeof(struct connectbundle));
if(!*cb_ptr)
return CURLE_OUT_OF_MEMORY;
(*cb_ptr)->num_connections = 0;
(*cb_ptr)->server_supports_pipelining = FALSE;
(*cb_ptr)->conn_list = Curl_llist_alloc((curl_llist_dtor) conn_llist_dtor);
if(!(*cb_ptr)->conn_list)
return CURLE_OUT_OF_MEMORY;
return CURLE_OK;
}
void Curl_bundle_destroy(struct connectbundle *cb_ptr)
{
if(cb_ptr->conn_list)
Curl_llist_destroy(cb_ptr->conn_list, NULL);
Curl_safefree(cb_ptr);
}
/* Add a connection to a bundle */
CURLcode Curl_bundle_add_conn(struct connectbundle *cb_ptr,
struct connectdata *conn)
{
if(!Curl_llist_insert_next(cb_ptr->conn_list, cb_ptr->conn_list->tail, conn))
return CURLE_OUT_OF_MEMORY;
conn->bundle = cb_ptr;
cb_ptr->num_connections++;
return CURLE_OK;
}
/* Remove a connection from a bundle */
int Curl_bundle_remove_conn(struct connectbundle *cb_ptr,
struct connectdata *conn)
{
struct curl_llist_element *curr;
curr = cb_ptr->conn_list->head;
while(curr) {
if(curr->ptr == conn) {
Curl_llist_remove(cb_ptr->conn_list, curr, NULL);
cb_ptr->num_connections--;
conn->bundle = NULL;
return 1; /* we removed a handle */
}
curr = curr->next;
}
return 0;
}

44
lib/bundles.h Normal file
View File

@ -0,0 +1,44 @@
#ifndef __BUNDLES_H
#define __BUNDLES_H
/***************************************************************************
* _ _ ____ _
* Project ___| | | | _ \| |
* / __| | | | |_) | |
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 2012, Linus Nielsen Feltzing, <linus@haxx.se>
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
* are also available at http://curl.haxx.se/docs/copyright.html.
*
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
* copies of the Software, and permit persons to whom the Software is
* furnished to do so, under the terms of the COPYING file.
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
* KIND, either express or implied.
*
***************************************************************************/
struct connectbundle {
bool server_supports_pipelining; /* TRUE if server supports pipelining,
set after first response */
size_t num_connections; /* Number of connections in the bundle */
struct curl_llist *conn_list; /* The connectdata members of the bundle */
};
CURLcode Curl_bundle_create(struct SessionHandle *data,
struct connectbundle **cb_ptr);
void Curl_bundle_destroy(struct connectbundle *cb_ptr);
CURLcode Curl_bundle_add_conn(struct connectbundle *cb_ptr,
struct connectdata *conn);
int Curl_bundle_remove_conn(struct connectbundle *cb_ptr,
struct connectdata *conn);
#endif /* __BUNDLES_H */

270
lib/conncache.c Normal file
View File

@ -0,0 +1,270 @@
/***************************************************************************
* _ _ ____ _
* Project ___| | | | _ \| |
* / __| | | | |_) | |
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 2012, Linus Nielsen Feltzing, <linus@haxx.se>
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
* are also available at http://curl.haxx.se/docs/copyright.html.
*
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
* copies of the Software, and permit persons to whom the Software is
* furnished to do so, under the terms of the COPYING file.
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
* KIND, either express or implied.
*
***************************************************************************/
#include "setup.h"
#include <curl/curl.h>
#include "urldata.h"
#include "url.h"
#include "progress.h"
#include "multiif.h"
#include "sendf.h"
#include "rawstr.h"
#include "bundles.h"
#include "conncache.h"
#include "curl_memory.h"
/* The last #include file should be: */
#include "memdebug.h"
#define CONNECTION_HASH_SIZE 97
static void free_bundle_hash_entry(void *freethis)
{
struct connectbundle *b = (struct connectbundle *) freethis;
Curl_bundle_destroy(b);
}
struct conncache *Curl_conncache_init(int type)
{
struct conncache *connc;
connc = calloc(1, sizeof(struct conncache));
if(!connc)
return NULL;
connc->hash = Curl_hash_alloc(CONNECTION_HASH_SIZE, Curl_hash_str,
Curl_str_key_compare, free_bundle_hash_entry);
if(!connc->hash) {
free(connc);
return NULL;
}
connc->type = type;
connc->num_connections = 0;
return connc;
}
void Curl_conncache_destroy(struct conncache *connc)
{
Curl_hash_destroy(connc->hash);
free(connc);
}
struct connectbundle *Curl_conncache_find_bundle(struct conncache *connc,
char *hostname)
{
struct connectbundle *bundle = NULL;
if(connc)
bundle = Curl_hash_pick(connc->hash, hostname, strlen(hostname)+1);
return bundle;
}
static bool conncache_add_bundle(struct conncache *connc,
char *hostname,
struct connectbundle *bundle)
{
void *p;
p = Curl_hash_add(connc->hash, hostname, strlen(hostname)+1, bundle);
return p?TRUE:FALSE;
}
static void conncache_remove_bundle(struct conncache *connc,
struct connectbundle *bundle)
{
struct curl_hash_iterator iter;
struct curl_hash_element *he;
if(!connc)
return;
Curl_hash_start_iterate(connc->hash, &iter);
he = Curl_hash_next_element(&iter);
while(he) {
if(he->ptr == bundle) {
/* The bundle is destroyed by the hash destructor function,
free_bundle_hash_entry() */
Curl_hash_delete(connc->hash, he->key, he->key_len);
return;
}
he = Curl_hash_next_element(&iter);
}
}
CURLcode Curl_conncache_add_conn(struct conncache *connc,
struct connectdata *conn)
{
CURLcode result;
struct connectbundle *bundle;
struct SessionHandle *data = conn->data;
bundle = Curl_conncache_find_bundle(data->state.conn_cache,
conn->host.name);
if(!bundle) {
result = Curl_bundle_create(data, &bundle);
if(result != CURLE_OK)
return result;
if(!conncache_add_bundle(data->state.conn_cache,
conn->host.name, bundle))
return CURLE_OUT_OF_MEMORY;
}
result = Curl_bundle_add_conn(bundle, conn);
if(result != CURLE_OK)
return result;
connc->num_connections++;
return CURLE_OK;
}
void Curl_conncache_remove_conn(struct conncache *connc,
struct connectdata *conn)
{
struct connectbundle *bundle = conn->bundle;
/* The bundle pointer can be NULL, since this function can be called
due to a failed connection attempt, before being added to a bundle */
if(bundle) {
Curl_bundle_remove_conn(bundle, conn);
if(bundle->num_connections == 0) {
conncache_remove_bundle(connc, bundle);
}
connc->num_connections--;
DEBUGF(infof(conn->data, "The cache now contains %d members\n",
connc->num_connections));
}
}
/* This function iterates the entire connection cache and calls the
function func() with the connection pointer as the first argument
and the supplied 'param' argument as the other */
void Curl_conncache_foreach(struct conncache *connc,
void *param,
void (*func)(void *conn, void *param))
{
struct curl_hash_iterator iter;
struct curl_llist_element *curr;
struct curl_hash_element *he;
if(!connc)
return;
Curl_hash_start_iterate(connc->hash, &iter);
he = Curl_hash_next_element(&iter);
while(he) {
struct connectbundle *bundle;
struct connectdata *conn;
bundle = he->ptr;
curr = bundle->conn_list->head;
while(curr) {
/* Yes, we need to update curr before calling func(), because func()
might decide to remove the connection */
conn = curr->ptr;
curr = curr->next;
func(conn, param);
}
he = Curl_hash_next_element(&iter);
}
}
/* Return the first connection found in the cache. Used when closing all
connections */
struct connectdata *
Curl_conncache_find_first_connection(struct conncache *connc)
{
struct curl_hash_iterator iter;
struct curl_llist_element *curr;
struct curl_hash_element *he;
struct connectbundle *bundle;
Curl_hash_start_iterate(connc->hash, &iter);
he = Curl_hash_next_element(&iter);
while(he) {
bundle = he->ptr;
curr = bundle->conn_list->head;
if(curr) {
return curr->ptr;
}
he = Curl_hash_next_element(&iter);
}
return NULL;
}
#if 0
/* Useful for debugging the connection cache */
void Curl_conncache_print(struct conncache *connc)
{
struct curl_hash_iterator iter;
struct curl_llist_element *curr;
struct curl_hash_element *he;
if(!connc)
return;
fprintf(stderr, "=Bundle cache=\n");
Curl_hash_start_iterate(connc->hash, &iter);
he = Curl_hash_next_element(&iter);
while(he) {
struct connectbundle *bundle;
struct connectdata *conn;
bundle = he->ptr;
fprintf(stderr, "%s -", he->key);
curr = bundle->conn_list->head;
while(curr) {
conn = curr->ptr;
fprintf(stderr, " [%p %d]", (void *)conn, conn->inuse);
curr = curr->next;
}
fprintf(stderr, "\n");
he = Curl_hash_next_element(&iter);
}
}
#endif

56
lib/conncache.h Normal file
View File

@ -0,0 +1,56 @@
#ifndef __CONNCACHE_H
#define __CONNCACHE_H
/***************************************************************************
* _ _ ____ _
* Project ___| | | | _ \| |
* / __| | | | |_) | |
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 2012, Linus Nielsen Feltzing, <linus@haxx.se>
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
* are also available at http://curl.haxx.se/docs/copyright.html.
*
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
* copies of the Software, and permit persons to whom the Software is
* furnished to do so, under the terms of the COPYING file.
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
* KIND, either express or implied.
*
***************************************************************************/
struct conncache {
struct curl_hash *hash;
enum {
CONNCACHE_PRIVATE, /* used for an easy handle alone */
CONNCACHE_MULTI /* shared within a multi handle */
} type;
size_t num_connections;
};
struct conncache *Curl_conncache_init(int type);
void Curl_conncache_destroy(struct conncache *connc);
struct connectbundle *Curl_conncache_find_bundle(struct conncache *connc,
char *hostname);
CURLcode Curl_conncache_add_conn(struct conncache *connc,
struct connectdata *conn);
void Curl_conncache_remove_conn(struct conncache *connc,
struct connectdata *conn);
void Curl_conncache_foreach(struct conncache *connc,
void *param,
void (*func)(void *, void *));
struct connectdata *
Curl_conncache_find_first_connection(struct conncache *connc);
void Curl_conncache_print(struct conncache *connc);
#endif /* __CONNCACHE_H */

6
lib/connect.c
View File

@ -1131,10 +1131,8 @@ curl_socket_t Curl_getconnectinfo(struct SessionHandle *data,
DEBUGASSERT(data);
if((data->state.lastconnect != -1) &&
(data->state.connc->connects[data->state.lastconnect] != NULL)) {
struct connectdata *c =
data->state.connc->connects[data->state.lastconnect];
if(data->state.lastconnect) {
struct connectdata *c = data->state.lastconnect;
if(connp)
/* only store this if the caller cares for it */
*connp = c;

18
lib/easy.c
View File

@ -70,6 +70,7 @@
#include "curl_rand.h"
#include "non-ascii.h"
#include "warnless.h"
#include "conncache.h"
#define _MPRINTF_REPLACE /* use our functions only */
#include <curl/mprintf.h>
@ -526,10 +527,10 @@ CURLcode curl_easy_perform(CURL *curl)
}
if(!data->state.connc) {
/* oops, no connection cache, make one up */
data->state.connc = Curl_mk_connc(CONNCACHE_PRIVATE, -1L);
if(!data->state.connc)
if(!data->state.conn_cache) {
/* Oops, no connection cache, create one */
data->state.conn_cache = Curl_conncache_init(CONNCACHE_PRIVATE);
if(!data->state.conn_cache)
return CURLE_OUT_OF_MEMORY;
}
@ -616,9 +617,9 @@ CURL *curl_easy_duphandle(CURL *incurl)
goto fail;
/* the connection cache is setup on demand */
outcurl->state.connc = NULL;
outcurl->state.conn_cache = NULL;
outcurl->state.lastconnect = -1;
outcurl->state.lastconnect = NULL;
outcurl->progress.flags = data->progress.flags;
outcurl->progress.callback = data->progress.callback;
@ -674,11 +675,6 @@ CURL *curl_easy_duphandle(CURL *incurl)
fail:
if(outcurl) {
if(outcurl->state.connc &&
(outcurl->state.connc->type == CONNCACHE_PRIVATE)) {
Curl_rm_connc(outcurl->state.connc);
outcurl->state.connc = NULL;
}
curl_slist_free_all(outcurl->change.cookielist);
outcurl->change.cookielist = NULL;
Curl_safefree(outcurl->state.headerbuff);

77
lib/hash.c
View File

@ -322,34 +322,77 @@ size_t Curl_str_key_compare(void*k1, size_t key1_len, void*k2, size_t key2_len)
return 0;
}
void Curl_hash_start_iterate(struct curl_hash *hash,
struct curl_hash_iterator *iter)
{
iter->hash = hash;
iter->slot_index = 0;
iter->current_element = NULL;
}
struct curl_hash_element *
Curl_hash_next_element(struct curl_hash_iterator *iter)
{
int i;
struct curl_hash *h = iter->hash;
/* Get the next element in the current list, if any */
if(iter->current_element)
iter->current_element = iter->current_element->next;
/* If we have reached the end of the list, find the next one */
if(!iter->current_element) {
for(i = iter->slot_index;i < h->slots;i++) {
if(h->table[i]->head) {
iter->current_element = h->table[i]->head;
iter->slot_index = i+1;
break;
}
}
}
if(iter->current_element) {
struct curl_hash_element *he = iter->current_element->ptr;
return he;
}
else {
iter->current_element = NULL;
return NULL;
}
}
#if 0 /* useful function for debugging hashes and their contents */
void Curl_hash_print(struct curl_hash *h,
void (*func)(void *))
{
int i;
struct curl_llist_element *le;
struct curl_llist *list;
struct curl_hash_element *he;
struct curl_hash_iterator iter;
struct curl_hash_element *he;
int last_index = -1;
if(!h)
return;
fprintf(stderr, "=Hash dump=\n");
for(i = 0; i < h->slots; i++) {
list = h->table[i];
le = list->head; /* get first list entry */
if(le) {
fprintf(stderr, "index %d:", i);
while(le) {
he = le->ptr;
if(func)
func(he->ptr);
else
fprintf(stderr, " [%p]", he->ptr);
le = le->next;
Curl_hash_start_iterate(h, &iter);
he = Curl_hash_next_element(&iter);
while(he) {
if(iter.slot_index != last_index) {
fprintf(stderr, "index %d:", iter.slot_index);
if(last_index >= 0) {
fprintf(stderr, "\n");
}
fprintf(stderr, "\n");
last_index = iter.slot_index;
}
if(func)
func(he->ptr);
else
fprintf(stderr, " [%p]", he->ptr);
he = Curl_hash_next_element(&iter);
}
fprintf(stderr, "\n");
}
#endif

14
lib/hash.h
View File

@ -62,6 +62,11 @@ struct curl_hash_element {
size_t key_len;
};
struct curl_hash_iterator {
struct curl_hash *hash;
int slot_index;
struct curl_llist_element *current_element;
};
int Curl_hash_init(struct curl_hash *h,
int slots,
@ -89,4 +94,13 @@ size_t Curl_hash_str(void* key, size_t key_length, size_t slots_num);
size_t Curl_str_key_compare(void*k1, size_t key1_len, void*k2,
size_t key2_len);
void Curl_hash_start_iterate(struct curl_hash *hash,
struct curl_hash_iterator *iter);
struct curl_hash_element *
Curl_hash_next_element(struct curl_hash_iterator *iter);
void Curl_hash_print(struct curl_hash *h,
void (*func)(void *));
#endif

1
lib/http.c
View File

@ -3170,6 +3170,7 @@ CURLcode Curl_http_readwrite_headers(struct SessionHandle *data,
}
else if(conn->httpversion >= 11 &&
!conn->bits.close) {
/* If HTTP version is >= 1.1 and connection is persistent
server supports pipelining. */
DEBUGF(infof(data,

313
lib/multi.c
View File

@ -44,6 +44,8 @@
#include "select.h"
#include "warnless.h"
#include "speedcheck.h"
#include "conncache.h"
#include "bundles.h"
#define _MPRINTF_REPLACE /* use our functions only */
#include <curl/mprintf.h>
@ -97,11 +99,6 @@ typedef enum {
#define GETSOCK_READABLE (0x00ff)
#define GETSOCK_WRITABLE (0xff00)
struct closure {
struct closure *next; /* a simple one-way list of structs */
struct SessionHandle *easy_handle;
};
struct Curl_one_easy {
/* first, two fields for the linked list of these */
struct Curl_one_easy *next;
@ -164,14 +161,16 @@ struct Curl_multi {
/* Whether pipelining is enabled for this multi handle */
bool pipelining_enabled;
/* shared connection cache */
struct conncache *connc;
/* Shared connection cache (bundles)*/
struct conncache *conn_cache;
/* This handle will be used for closing the cached connections in
curl_multi_cleanup() */
struct SessionHandle *closure_handle;
long maxconnects; /* if >0, a fixed limit of the maximum number of entries
we're allowed to grow the connection cache to */
/* list of easy handles kept around for doing nice connection closures */
struct closure *closure;
/* timer callback and user data pointer for the *socket() API */
curl_multi_timer_callback timer_cb;
void *timer_userp;
@ -179,12 +178,8 @@ struct Curl_multi {
previous callback */
};
static void multi_connc_remove_handle(struct Curl_multi *multi,
struct SessionHandle *data);
static void singlesocket(struct Curl_multi *multi,
struct Curl_one_easy *easy);
static CURLMcode add_closure(struct Curl_multi *multi,
struct SessionHandle *data);
static int update_timer(struct Curl_multi *multi);
static CURLcode addHandleToSendOrPendPipeline(struct SessionHandle *handle,
@ -228,7 +223,7 @@ static void multi_freetimeout(void *a, void *b);
static void multistate(struct Curl_one_easy *easy, CURLMstate state)
{
#ifdef DEBUGBUILD
long connectindex = -5000;
long connection_id = -5000;
#endif
CURLMstate oldstate = easy->state;
@ -242,12 +237,12 @@ static void multistate(struct Curl_one_easy *easy, CURLMstate state)
if(easy->easy_conn) {
if(easy->state > CURLM_STATE_CONNECT &&
easy->state < CURLM_STATE_COMPLETED)
connectindex = easy->easy_conn->connectindex;
connection_id = easy->easy_conn->connection_id;
infof(easy->easy_handle,
"STATE: %s => %s handle %p; (connection #%ld) \n",
statename[oldstate], statename[easy->state],
(char *)easy, connectindex);
(char *)easy, connection_id);
}
#endif
if(state == CURLM_STATE_COMPLETED)
@ -391,7 +386,6 @@ static void multi_freeamsg(void *a, void *b)
(void)b;
}
CURLM *curl_multi_init(void)
{
struct Curl_multi *multi = calloc(1, sizeof(struct Curl_multi));
@ -409,8 +403,8 @@ CURLM *curl_multi_init(void)
if(!multi->sockhash)
goto error;
multi->connc = Curl_mk_connc(CONNCACHE_MULTI, -1L);
if(!multi->connc)
multi->conn_cache = Curl_conncache_init(CONNCACHE_MULTI);
if(!multi->conn_cache)
goto error;
multi->msglist = Curl_llist_alloc(multi_freeamsg);
@ -431,8 +425,8 @@ CURLM *curl_multi_init(void)
multi->sockhash = NULL;
Curl_hash_destroy(multi->hostcache);
multi->hostcache = NULL;
Curl_rm_connc(multi->connc);
multi->connc = NULL;
Curl_conncache_destroy(multi->conn_cache);
multi->conn_cache = NULL;
free(multi);
return NULL;
@ -443,8 +437,6 @@ CURLMcode curl_multi_add_handle(CURLM *multi_handle,
{
struct curl_llist *timeoutlist;
struct Curl_one_easy *easy;
struct closure *cl;
struct closure *prev = NULL;
struct Curl_multi *multi = (struct Curl_multi *)multi_handle;
struct SessionHandle *data = (struct SessionHandle *)easy_handle;
@ -462,22 +454,13 @@ CURLMcode curl_multi_add_handle(CURLM *multi_handle,
/* possibly we should create a new unique error code for this condition */
return CURLM_BAD_EASY_HANDLE;
/* We want the connection cache to have plenty of room. Before we supported
the shared cache every single easy handle had 5 entries in their cache
by default. */
if(((multi->num_easy + 1) * 4) > multi->connc->num) {
long newmax = (multi->num_easy + 1) * 4;
if(multi->maxconnects && (newmax > multi->maxconnects))
/* don't grow beyond the allowed size */
newmax = multi->maxconnects;
if(newmax > multi->connc->num) {
/* we only do this is we can in fact grow the cache */
CURLcode res = Curl_ch_connc(data, multi->connc, newmax);
if(res)
return CURLM_OUT_OF_MEMORY;
}
/* This is a good time to allocate a fresh easy handle to use when closing
cached connections */
if(!multi->closure_handle) {
multi->closure_handle =
(struct SessionHandle *)curl_easy_init();
Curl_easy_addmulti(easy_handle, multi_handle);
multi->closure_handle->state.conn_cache = multi->conn_cache;
}
/* Allocate and initialize timeout list for easy handle */
@ -504,27 +487,6 @@ CURLMcode curl_multi_add_handle(CURLM *multi_handle,
data->state.timeoutlist = timeoutlist;
timeoutlist = NULL;
/* Remove handle from the list of 'closure handles' in case it is there */
cl = multi->closure;
while(cl) {
struct closure *next = cl->next;
if(cl->easy_handle == data) {
/* Remove node from list */
free(cl);
if(prev)
prev->next = next;
else
multi->closure = next;
/* removed from closure list now, this might reuse an existing
existing connection but we don't know that at this point */
data->state.shared_conn = NULL;
/* No need to continue, handle can only be present once in the list */
break;
}
prev = cl;
cl = next;
}
/* set the easy handle */
easy->easy_handle = data;
multistate(easy, CURLM_STATE_INIT);
@ -548,16 +510,15 @@ CURLMcode curl_multi_add_handle(CURLM *multi_handle,
}
/* On a multi stack the connection cache, owned by the multi handle,
is shared between all easy handles within the multi handle. */
if(easy->easy_handle->state.connc &&
(easy->easy_handle->state.connc->type == CONNCACHE_PRIVATE)) {
/* kill old private connection cache */
Curl_rm_connc(easy->easy_handle->state.connc);
easy->easy_handle->state.connc = NULL;
is shared between all easy handles within the multi handle.
Therefore we free the private connection cache if there is one */
if(easy->easy_handle->state.conn_cache &&
easy->easy_handle->state.conn_cache->type == CONNCACHE_PRIVATE) {
Curl_conncache_destroy(easy->easy_handle->state.conn_cache);
}
/* Point now to this multi's connection cache */
easy->easy_handle->state.connc = multi->connc;
easy->easy_handle->state.connc->type = CONNCACHE_MULTI;
easy->easy_handle->state.conn_cache = multi->conn_cache;
/* This adds the new entry at the 'end' of the doubly-linked circular
list of Curl_one_easy structs to try and maintain a FIFO queue so
@ -701,42 +662,17 @@ CURLMcode curl_multi_remove_handle(CURLM *multi_handle,
Note that this ignores the return code simply because there's
nothing really useful to do with it anyway! */
(void)Curl_done(&easy->easy_conn, easy->result, premature);
if(easy->easy_conn)
/* the connection is still alive, set back the association to enable
the check below to trigger TRUE */
easy->easy_conn->data = easy->easy_handle;
}
else
/* Clear connection pipelines, if Curl_done above was not called */
Curl_getoff_all_pipelines(easy->easy_handle, easy->easy_conn);
}
/* figure out if the easy handle is used by one or more connections in the
cache */
multi_connc_remove_handle(multi, easy->easy_handle);
if(easy->easy_handle->state.connc->type == CONNCACHE_MULTI) {
if(easy->easy_handle->state.conn_cache->type == CONNCACHE_MULTI) {
/* if this was using the shared connection cache we clear the pointer
to that since we're not part of that handle anymore */
easy->easy_handle->state.connc = NULL;
/* Since we return the connection back to the communal connection pool
we mark the last connection as inaccessible */
easy->easy_handle->state.lastconnect = -1;
/* Modify the connectindex since this handle can't point to the
connection cache anymore.
TODO: consider if this is really what we want. The connection cache
is within the multi handle and that owns the connections so we should
not need to touch connections like this when we just remove an easy
handle...
*/
if(easy->easy_conn && easy_owns_conn &&
(easy->easy_conn->send_pipe->size +
easy->easy_conn->recv_pipe->size == 0))
easy->easy_conn->connectindex = -1;
easy->easy_handle->state.conn_cache = NULL;
easy->easy_handle->state.lastconnect = NULL;
}
/* change state without using multistate(), only to make singlesocket() do
@ -745,6 +681,12 @@ CURLMcode curl_multi_remove_handle(CURLM *multi_handle,
singlesocket(multi, easy); /* to let the application know what sockets
that vanish with this handle */
/* Remove the association between the connection and the handle */
if(easy->easy_conn) {
easy->easy_conn->data = NULL;
easy->easy_conn = NULL;
}
Curl_easy_addmulti(easy->easy_handle, NULL); /* clear the association
to this multi handle */
@ -1324,8 +1266,8 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
case CURLM_STATE_WAITDO:
/* Wait for our turn to DO when we're pipelining requests */
#ifdef DEBUGBUILD
infof(data, "Conn %ld send pipe %zu inuse %d athead %d\n",
easy->easy_conn->connectindex,
infof(data, "WAITDO: Conn %ld send pipe %zu inuse %d athead %d\n",
easy->easy_conn->connection_id,
easy->easy_conn->send_pipe->size,
easy->easy_conn->writechannel_inuse?1:0,
isHandleAtHead(data,
@ -1514,8 +1456,8 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
}
#ifdef DEBUGBUILD
else {
infof(data, "Conn %ld recv pipe %zu inuse %d athead %d\n",
easy->easy_conn->connectindex,
infof(data, "WAITPERFORM: Conn %ld recv pipe %zu inuse %d athead %d\n",
easy->easy_conn->connection_id,
easy->easy_conn->recv_pipe->size,
easy->easy_conn->readchannel_inuse?1:0,
isHandleAtHead(data,
@ -1891,45 +1833,41 @@ CURLMcode curl_multi_perform(CURLM *multi_handle, int *running_handles)
return returncode;
}
static void close_all_connections(struct Curl_multi *multi)
{
struct connectdata *conn;
conn = Curl_conncache_find_first_connection(multi->conn_cache);
while(conn) {
conn->data = multi->closure_handle;
/* This will remove the connection from the cache */
(void)Curl_disconnect(conn, FALSE);
conn = Curl_conncache_find_first_connection(multi->conn_cache);
}
}
CURLMcode curl_multi_cleanup(CURLM *multi_handle)
{
struct Curl_multi *multi=(struct Curl_multi *)multi_handle;
struct Curl_one_easy *easy;
struct Curl_one_easy *nexteasy;
int i;
struct closure *cl;
struct closure *n;
if(GOOD_MULTI_HANDLE(multi)) {
multi->type = 0; /* not good anymore */
/* go over all connections that have close actions */
for(i=0; i< multi->connc->num; i++) {
if(multi->connc->connects[i] &&
multi->connc->connects[i]->handler->flags & PROTOPT_CLOSEACTION) {
Curl_disconnect(multi->connc->connects[i], FALSE);
multi->connc->connects[i] = NULL;
}
}
/* now walk through the list of handles we kept around only to be
able to close connections "properly" */
cl = multi->closure;
while(cl) {
cl->easy_handle->state.shared_conn = NULL; /* allow cleanup */
if(cl->easy_handle->state.closed)
/* close handle only if curl_easy_cleanup() already has been called
for this easy handle */
Curl_close(cl->easy_handle);
n = cl->next;
free(cl);
cl= n;
}
/* Close all the connections in the connection cache */
close_all_connections(multi);
Curl_close(multi->closure_handle);
multi->closure_handle = NULL;
Curl_hash_destroy(multi->sockhash);
multi->sockhash = NULL;
Curl_rm_connc(multi->connc);
multi->connc = NULL;
Curl_conncache_destroy(multi->conn_cache);
multi->conn_cache = NULL;
/* remove the pending list of messages */
Curl_llist_destroy(multi->msglist, NULL);
@ -1947,7 +1885,7 @@ CURLMcode curl_multi_cleanup(CURLM *multi_handle)
}
/* Clear the pointer to the connection cache */
easy->easy_handle->state.connc = NULL;
easy->easy_handle->state.conn_cache = NULL;
Curl_easy_addmulti(easy->easy_handle, NULL); /* clear the association */
@ -2803,125 +2741,6 @@ CURLMcode curl_multi_assign(CURLM *multi_handle,
return CURLM_OK;
}
static void multi_connc_remove_handle(struct Curl_multi *multi,
struct SessionHandle *data)
{
/* a connection in the connection cache pointing to the given 'data' ? */
int i;
for(i=0; i< multi->connc->num; i++) {
struct connectdata * conn = multi->connc->connects[i];
if(conn && conn->data == data) {
/* If this easy_handle was the last one in charge for one or more
connections in the shared connection cache, we might need to keep
this handle around until either A) the connection is closed and
killed properly, or B) another easy_handle uses the connection.
The reason why we need to have a easy_handle associated with a live
connection is simply that some connections will need a handle to get
closed down properly. Currently, the only connections that need to
keep a easy_handle handle around are using FTP(S). Such connections
have the PROT_CLOSEACTION bit set.
Thus, we need to check for all connections in the shared cache that
points to this handle and are using PROT_CLOSEACTION. If there's any,
we need to add this handle to the list of "easy handles kept around
for nice connection closures".
*/
if(conn->handler->flags & PROTOPT_CLOSEACTION) {
/* this handle is still being used by a shared connection and
thus we leave it around for now */
if(add_closure(multi, data) == CURLM_OK)
data->state.shared_conn = multi;
else {
/* out of memory - so much for graceful shutdown */
Curl_disconnect(conn, /* dead_connection */ FALSE);
multi->connc->connects[i] = NULL;
data->state.shared_conn = NULL;
}
}
else {
/* disconect the easy handle from the connection since the connection
will now remain but this easy handle is going */
data->state.shared_conn = NULL;
conn->data = NULL;
}
}
}
}
/* Add the given data pointer to the list of 'closure handles' that are kept
around only to be able to close some connections nicely - just make sure
that this handle isn't already added, like for the cases when an easy
handle is removed, added and removed again... */
static CURLMcode add_closure(struct Curl_multi *multi,
struct SessionHandle *data)
{
struct closure *cl = multi->closure;
struct closure *p = NULL;
bool add = TRUE;
/* Before adding, scan through all the other currently kept handles and see
if there are any connections still referring to them and kill them if
not. */
while(cl) {
struct closure *n;
bool inuse = FALSE;
int i;
for(i=0; i< multi->connc->num; i++) {
if(multi->connc->connects[i] &&
(multi->connc->connects[i]->data == cl->easy_handle)) {
inuse = TRUE;
break;
}
}
n = cl->next;
if(!inuse) {
/* cl->easy_handle is now killable */
/* unmark it as not having a connection around that uses it anymore */
cl->easy_handle->state.shared_conn= NULL;
if(cl->easy_handle->state.closed) {
infof(data, "Delayed kill of easy handle %p\n", cl->easy_handle);
/* close handle only if curl_easy_cleanup() already has been called
for this easy handle */
Curl_close(cl->easy_handle);
}
if(p)
p->next = n;
else
multi->closure = n;
free(cl);
}
else {
if(cl->easy_handle == data)
add = FALSE;
p = cl;
}
cl = n;
}
if(add) {
cl = calloc(1, sizeof(struct closure));
if(!cl)
return CURLM_OUT_OF_MEMORY;
cl->easy_handle = data;
cl->next = multi->closure;
multi->closure = cl;
}
return CURLM_OK;
}
#ifdef DEBUGBUILD
void Curl_multi_dump(const struct Curl_multi *multi_handle)
{

708
lib/url.c
View File

@ -126,6 +126,8 @@ int curl_win32_idn_to_ascii(const char *in, char **out);
#include "curl_rtmp.h"
#include "gopher.h"
#include "http_proxy.h"
#include "bundles.h"
#include "conncache.h"
#define _MPRINTF_REPLACE /* use our functions only */
#include <curl/mprintf.h>
@ -135,7 +137,7 @@ int curl_win32_idn_to_ascii(const char *in, char **out);
#include "memdebug.h"
/* Local static prototypes */
static long ConnectionKillOne(struct SessionHandle *data);
static bool ConnectionKillOne(struct SessionHandle *data);
static void conn_free(struct connectdata *conn);
static void signalPipeClose(struct curl_llist *pipeline, bool pipe_broke);
static CURLcode do_init(struct connectdata *conn);
@ -262,10 +264,10 @@ static const struct Curl_handler Curl_handler_dummy = {
static void close_connections(struct SessionHandle *data)
{
/* Loop through all open connections and kill them one by one */
long i;
bool killed;
do {
i = ConnectionKillOne(data);
} while(i != -1L);
killed = ConnectionKillOne(data);
} while(killed);
}
void Curl_freeset(struct SessionHandle * data)
@ -378,66 +380,6 @@ CURLcode Curl_close(struct SessionHandle *data)
{
struct Curl_multi *m = data->multi;
#ifdef DEBUGBUILD
/* only for debugging, scan through all connections and see if there's a
pipe reference still identifying this handle */
if(data->state.connc && data->state.connc->type == CONNCACHE_MULTI) {
struct conncache *c = data->state.connc;
long i;
struct curl_llist *pipeline;
struct curl_llist_element *curr;
struct connectdata *connptr;
for(i=0; i< c->num; i++) {
connptr = c->connects[i];
if(!connptr)
continue;
pipeline = connptr->send_pipe;
if(pipeline) {
for(curr = pipeline->head; curr; curr=curr->next) {
if(data == (struct SessionHandle *) curr->ptr) {
fprintf(stderr,
"problem we %p are still in send pipe for %p done %d\n",
data, connptr, (int)connptr->bits.done);
}
}
}
pipeline = connptr->recv_pipe;
if(pipeline) {
for(curr = pipeline->head; curr; curr=curr->next) {
if(data == (struct SessionHandle *) curr->ptr) {
fprintf(stderr,
"problem we %p are still in recv pipe for %p done %d\n",
data, connptr, (int)connptr->bits.done);
}
}
}
pipeline = connptr->done_pipe;
if(pipeline) {
for(curr = pipeline->head; curr; curr=curr->next) {
if(data == (struct SessionHandle *) curr->ptr) {
fprintf(stderr,
"problem we %p are still in done pipe for %p done %d\n",
data, connptr, (int)connptr->bits.done);
}
}
}
pipeline = connptr->pend_pipe;
if(pipeline) {
for(curr = pipeline->head; curr; curr=curr->next) {
if(data == (struct SessionHandle *) curr->ptr) {
fprintf(stderr,
"problem we %p are still in pend pipe for %p done %d\n",
data, connptr, (int)connptr->bits.done);
}
}
}
}
}
#endif
Curl_expire(data, 0); /* shut off timers */
if(m)
@ -457,26 +399,16 @@ CURLcode Curl_close(struct SessionHandle *data)
the multi handle, since that function uses the magic
field! */
if(data->state.connc) {
if(data->state.connc->type == CONNCACHE_PRIVATE) {
if(data->state.conn_cache) {
if(data->state.conn_cache->type == CONNCACHE_PRIVATE) {
/* close all connections still alive that are in the private connection
cache, as we no longer have the pointer left to the shared one. */
close_connections(data);
/* free the connection cache if allocated privately */
Curl_rm_connc(data->state.connc);
data->state.connc = NULL;
Curl_conncache_destroy(data->state.conn_cache);
data->state.conn_cache = NULL;
}
}
if(data->state.shared_conn) {
/* marked to be used by a pending connection so we can't kill this handle
just yet */
data->state.closed = TRUE;
return CURLE_OK;
}
if(data->dns.hostcachetype == HCACHE_PRIVATE)
Curl_hostcache_destroy(data);
@ -533,124 +465,6 @@ CURLcode Curl_close(struct SessionHandle *data)
return CURLE_OK;
}
/* create a connection cache of a private or multi type */
struct conncache *Curl_mk_connc(int type,
long amount) /* set -1 to use default */
{
/* It is subject for debate how many default connections to have for a multi
connection cache... */
struct conncache *c;
long default_amount;
long max_amount = (long)(((size_t)INT_MAX) / sizeof(struct connectdata *));
if(type == CONNCACHE_PRIVATE) {
default_amount = (amount < 1L) ? 5L : amount;
}
else {
default_amount = (amount < 1L) ? 10L : amount;
}
if(default_amount > max_amount)
default_amount = max_amount;
c = calloc(1, sizeof(struct conncache));
if(!c)
return NULL;
c->connects = calloc((size_t)default_amount, sizeof(struct connectdata *));
if(!c->connects) {
free(c);
return NULL;
}
c->num = default_amount;
return c;
}
/* Change number of entries of a connection cache */
CURLcode Curl_ch_connc(struct SessionHandle *data,
struct conncache *c,
long newamount)
{
long i;
struct connectdata **newptr;
long max_amount = (long)(((size_t)INT_MAX) / sizeof(struct connectdata *));
if(newamount < 1)
newamount = 1; /* we better have at least one entry */
if(!c) {
/* we get a NULL pointer passed in as connection cache, which means that
there is no cache created for this SessionHandle just yet, we create a
brand new with the requested size.
*/
data->state.connc = Curl_mk_connc(CONNCACHE_PRIVATE, newamount);
if(!data->state.connc)
return CURLE_OUT_OF_MEMORY;
return CURLE_OK;
}
if(newamount < c->num) {
/* Since this number is *decreased* from the existing number, we must
close the possibly open connections that live on the indexes that
are being removed!
NOTE: for conncache_multi cases we must make sure that we only
close handles not in use.
*/
for(i=newamount; i< c->num; i++) {
Curl_disconnect(c->connects[i], /* dead_connection */ FALSE);
c->connects[i] = NULL;
}
/* If the most recent connection is no longer valid, mark it
invalid. */
if(data->state.lastconnect <= newamount)
data->state.lastconnect = -1;
}
if(newamount > 0) {
if(newamount > max_amount)
newamount = max_amount;
newptr = realloc(c->connects, sizeof(struct connectdata *) * newamount);
if(!newptr)
/* we closed a few connections in vain, but so what? */
return CURLE_OUT_OF_MEMORY;
/* nullify the newly added pointers */
for(i=c->num; i<newamount; i++)
newptr[i] = NULL;
c->connects = newptr;
c->num = newamount;
}
/* we no longer support less than 1 as size for the connection cache, and
I'm not sure it ever worked to set it to zero */
return CURLE_OK;
}
/* Free a connection cache. This is called from Curl_close() and
curl_multi_cleanup(). */
void Curl_rm_connc(struct conncache *c)
{
if(!c)
return;
if(c->connects) {
long i;
for(i = 0; i < c->num; ++i) {
conn_free(c->connects[i]);
c->connects[i] = NULL;
}
free(c->connects);
c->connects = NULL;
}
c->num = 0;
free(c);
}
/*
* Initialize the UserDefined fields within a SessionHandle.
* This may be safely called on a new or existing SessionHandle.
@ -807,7 +621,7 @@ CURLcode Curl_open(struct SessionHandle **curl)
Curl_convert_init(data);
/* most recent connection is not yet defined */
data->state.lastconnect = -1;
data->state.lastconnect = NULL;
data->progress.flags |= PGRS_HIDE;
data->state.current_speed = -1; /* init to negative == impossible */
@ -880,7 +694,7 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option,
* Set the absolute number of maximum simultaneous alive connection that
* libcurl is allowed to have.
*/
result = Curl_ch_connc(data, data->state.connc, va_arg(param, long));
data->set.maxconnects = va_arg(param, long);
break;
case CURLOPT_FORBID_REUSE:
/*
@ -2744,14 +2558,9 @@ CURLcode Curl_disconnect(struct connectdata *conn, bool dead_connection)
/* This is set if protocol-specific cleanups should be made */
conn->handler->disconnect(conn, dead_connection);
if(-1 != conn->connectindex) {
/* unlink ourselves! */
infof(data, "Closing connection #%ld\n", conn->connectindex);
if(data->state.connc)
/* only clear the table entry if we still know in which cache we
used to be in */
data->state.connc->connects[conn->connectindex] = NULL;
}
infof(data, "Closing connection %d\n", conn->connection_id);
Curl_conncache_remove_conn(data->state.conn_cache, conn);
#if defined(USE_LIBIDN)
if(conn->host.encalloc)
@ -2938,225 +2747,229 @@ ConnectionExists(struct SessionHandle *data,
struct connectdata *needle,
struct connectdata **usethis)
{
long i;
struct connectdata *check;
struct connectdata *chosen = 0;
bool canPipeline = IsPipeliningPossible(data, needle);
bool wantNTLM = (data->state.authhost.want==CURLAUTH_NTLM) ||
(data->state.authhost.want==CURLAUTH_NTLM_WB) ? TRUE : FALSE;
struct connectbundle *bundle;
for(i=0; i< data->state.connc->num; i++) {
bool match = FALSE;
bool credentialsMatch = FALSE;
size_t pipeLen = 0;
/*
* Note that if we use a HTTP proxy, we check connections to that
* proxy and not to the actual remote server.
*/
check = data->state.connc->connects[i];
if(!check)
/* NULL pointer means not filled-in entry */
continue;
/* Look up the bundle with all the connections to this
particular host */
bundle = Curl_conncache_find_bundle(data->state.conn_cache,
needle->host.name);
if(bundle) {
struct curl_llist_element *curr;
pipeLen = check->send_pipe->size + check->recv_pipe->size;
infof(data, "Found bundle for host %s: %p\n", needle->host.name, bundle);
if(check->connectindex == -1) {
check->connectindex = i; /* Set this appropriately since it might have
been set to -1 when the easy was removed
from the multi */
}
curr = bundle->conn_list->head;
while(curr) {
bool match = FALSE;
bool credentialsMatch = FALSE;
size_t pipeLen;
if(!pipeLen && !check->inuse) {
/* The check for a dead socket makes sense only if there are no
handles in pipeline and the connection isn't already marked in
use */
bool dead;
if(check->handler->protocol & CURLPROTO_RTSP)
/* RTSP is a special case due to RTP interleaving */
dead = Curl_rtsp_connisdead(check);
else
dead = SocketIsDead(check->sock[FIRSTSOCKET]);
/*
* Note that if we use a HTTP proxy, we check connections to that
* proxy and not to the actual remote server.
*/
check = curr->ptr;
curr = curr->next;
if(dead) {
check->data = data;
infof(data, "Connection #%ld seems to be dead!\n", i);
pipeLen = check->send_pipe->size + check->recv_pipe->size;
/* disconnect resources */
Curl_disconnect(check, /* dead_connection */ TRUE);
data->state.connc->connects[i]=NULL; /* nothing here */
if(!pipeLen && !check->inuse) {
/* The check for a dead socket makes sense only if there are no
handles in pipeline and the connection isn't already marked in
use */
bool dead;
if(check->handler->protocol & CURLPROTO_RTSP)
/* RTSP is a special case due to RTP interleaving */
dead = Curl_rtsp_connisdead(check);
else
dead = SocketIsDead(check->sock[FIRSTSOCKET]);
continue;
}
}
if(dead) {
check->data = data;
infof(data, "Connection %d seems to be dead!\n",
check->connection_id);
if(canPipeline) {
/* Make sure the pipe has only GET requests */
struct SessionHandle* sh = gethandleathead(check->send_pipe);
struct SessionHandle* rh = gethandleathead(check->recv_pipe);
if(sh) {
if(!IsPipeliningPossible(sh, check))
continue;
}
else if(rh) {
if(!IsPipeliningPossible(rh, check))
/* disconnect resources */
Curl_disconnect(check, /* dead_connection */ TRUE);
continue;
}
}
if(canPipeline) {
/* Make sure the pipe has only GET requests */
struct SessionHandle* sh = gethandleathead(check->send_pipe);
struct SessionHandle* rh = gethandleathead(check->recv_pipe);
if(sh) {
if(!IsPipeliningPossible(sh, check))
continue;
}
else if(rh) {
if(!IsPipeliningPossible(rh, check))
continue;
}
#ifdef DEBUGBUILD
if(pipeLen > MAX_PIPELINE_LENGTH) {
infof(data, "BAD! Connection #%ld has too big pipeline!\n",
check->connectindex);
check->connection_id);
}
#endif
}
else {
if(pipeLen > 0) {
/* can only happen within multi handles, and means that another easy
handle is using this connection */
continue;
}
else {
if(pipeLen > 0) {
/* can only happen within multi handles, and means that another easy
handle is using this connection */
continue;
}
if(Curl_resolver_asynch()) {
/* ip_addr_str[0] is NUL only if the resolving of the name hasn't
completed yet and until then we don't re-use this connection */
if(!check->ip_addr_str[0]) {
infof(data,
"Connection #%ld hasn't finished name resolve, can't reuse\n",
check->connectindex);
if(Curl_resolver_asynch()) {
/* ip_addr_str[0] is NUL only if the resolving of the name hasn't
completed yet and until then we don't re-use this connection */
if(!check->ip_addr_str[0]) {
infof(data,
"Connection #%ld is still name resolving, can't reuse\n",
check->connection_id);
continue;
}
}
if((check->sock[FIRSTSOCKET] == CURL_SOCKET_BAD) ||
check->bits.close) {
/* Don't pick a connection that hasn't connected yet or that is going
to get closed. */
infof(data, "Connection #%ld isn't open enough, can't reuse\n",
check->connection_id);
#ifdef DEBUGBUILD
if(check->recv_pipe->size > 0) {
infof(data,
"BAD! Unconnected #%ld has a non-empty recv pipeline!\n",
check->connection_id);
}
#endif
continue;
}
}
if((check->sock[FIRSTSOCKET] == CURL_SOCKET_BAD) || check->bits.close) {
/* Don't pick a connection that hasn't connected yet or that is going
to get closed. */
infof(data, "Connection #%ld isn't open enough, can't reuse\n",
check->connectindex);
#ifdef DEBUGBUILD
if(check->recv_pipe->size > 0) {
infof(data, "BAD! Unconnected #%ld has a non-empty recv pipeline!\n",
check->connectindex);
}
#endif
continue;
if((needle->handler->flags&PROTOPT_SSL) !=
(check->handler->flags&PROTOPT_SSL))
/* don't do mixed SSL and non-SSL connections */
if(!(needle->handler->protocol & check->handler->protocol))
/* except protocols that have been upgraded via TLS */
continue;
if(needle->handler->flags&PROTOPT_SSL) {
if((data->set.ssl.verifypeer != check->verifypeer) ||
(data->set.ssl.verifyhost != check->verifyhost))
continue;
}
}
if((needle->handler->flags&PROTOPT_SSL) !=
(check->handler->flags&PROTOPT_SSL))
/* don't do mixed SSL and non-SSL connections */
if(!(needle->handler->protocol & check->handler->protocol))
/* except protocols that have been upgraded via TLS */
if(needle->bits.proxy != check->bits.proxy)
/* don't do mixed proxy and non-proxy connections */
continue;
if(needle->handler->flags&PROTOPT_SSL) {
if((data->set.ssl.verifypeer != check->verifypeer) ||
(data->set.ssl.verifyhost != check->verifyhost))
if(!canPipeline && check->inuse)
/* this request can't be pipelined but the checked connection is
already in use so we skip it */
continue;
}
if(needle->bits.proxy != check->bits.proxy)
/* don't do mixed proxy and non-proxy connections */
continue;
if(needle->localdev || needle->localport) {
/* If we are bound to a specific local end (IP+port), we must not
re-use a random other one, although if we didn't ask for a
particular one we can reuse one that was bound.
if(!canPipeline && check->inuse)
/* this request can't be pipelined but the checked connection is already
in use so we skip it */
continue;
if(needle->localdev || needle->localport) {
/* If we are bound to a specific local end (IP+port), we must not re-use
a random other one, although if we didn't ask for a particular one we
can reuse one that was bound.
This comparison is a bit rough and too strict. Since the input
parameters can be specified in numerous ways and still end up the
same it would take a lot of processing to make it really accurate.
Instead, this matching will assume that re-uses of bound connections
will most likely also re-use the exact same binding parameters and
missing out a few edge cases shouldn't hurt anyone very much.
*/
if((check->localport != needle->localport) ||
(check->localportrange != needle->localportrange) ||
!check->localdev ||
!needle->localdev ||
strcmp(check->localdev, needle->localdev))
continue;
}
if(!needle->bits.httpproxy || needle->handler->flags&PROTOPT_SSL ||
(needle->bits.httpproxy && check->bits.httpproxy &&
needle->bits.tunnel_proxy && check->bits.tunnel_proxy &&
Curl_raw_equal(needle->proxy.name, check->proxy.name) &&
(needle->port == check->port))) {
/* The requested connection does not use a HTTP proxy or it uses SSL or
it is a non-SSL protocol tunneled over the same http proxy name and
port number or it is a non-SSL protocol which is allowed to be
upgraded via TLS */
if((Curl_raw_equal(needle->handler->scheme, check->handler->scheme) ||
needle->handler->protocol & check->handler->protocol) &&
Curl_raw_equal(needle->host.name, check->host.name) &&
needle->remote_port == check->remote_port) {
if(needle->handler->flags & PROTOPT_SSL) {
/* This is a SSL connection so verify that we're using the same
SSL options as well */
if(!Curl_ssl_config_matches(&needle->ssl_config,
&check->ssl_config)) {
DEBUGF(infof(data,
"Connection #%ld has different SSL parameters, "
"can't reuse\n",
check->connectindex));
continue;
}
else if(check->ssl[FIRSTSOCKET].state != ssl_connection_complete) {
DEBUGF(infof(data,
"Connection #%ld has not started SSL connect, "
"can't reuse\n",
check->connectindex));
continue;
}
}
if((needle->handler->protocol & CURLPROTO_FTP) ||
((needle->handler->protocol & CURLPROTO_HTTP) && wantNTLM)) {
/* This is FTP or HTTP+NTLM, verify that we're using the same name
and password as well */
if(!strequal(needle->user, check->user) ||
!strequal(needle->passwd, check->passwd)) {
/* one of them was different */
continue;
}
credentialsMatch = TRUE;
}
match = TRUE;
This comparison is a bit rough and too strict. Since the input
parameters can be specified in numerous ways and still end up the
same it would take a lot of processing to make it really accurate.
Instead, this matching will assume that re-uses of bound connections
will most likely also re-use the exact same binding parameters and
missing out a few edge cases shouldn't hurt anyone very much.
*/
if((check->localport != needle->localport) ||
(check->localportrange != needle->localportrange) ||
!check->localdev ||
!needle->localdev ||
strcmp(check->localdev, needle->localdev))
continue;
}
}
else { /* The requested needle connection is using a proxy,
is the checked one using the same host, port and type? */
if(check->bits.proxy &&
(needle->proxytype == check->proxytype) &&
(needle->bits.tunnel_proxy == check->bits.tunnel_proxy) &&
Curl_raw_equal(needle->proxy.name, check->proxy.name) &&
needle->port == check->port) {
/* This is the same proxy connection, use it! */
match = TRUE;
if(!needle->bits.httpproxy || needle->handler->flags&PROTOPT_SSL ||
(needle->bits.httpproxy && check->bits.httpproxy &&
needle->bits.tunnel_proxy && check->bits.tunnel_proxy &&
Curl_raw_equal(needle->proxy.name, check->proxy.name) &&
(needle->port == check->port))) {
/* The requested connection does not use a HTTP proxy or it uses SSL or
it is a non-SSL protocol tunneled over the same http proxy name and
port number or it is a non-SSL protocol which is allowed to be
upgraded via TLS */
if((Curl_raw_equal(needle->handler->scheme, check->handler->scheme) ||
needle->handler->protocol & check->handler->protocol) &&
Curl_raw_equal(needle->host.name, check->host.name) &&
needle->remote_port == check->remote_port) {
if(needle->handler->flags & PROTOPT_SSL) {
/* This is a SSL connection so verify that we're using the same
SSL options as well */
if(!Curl_ssl_config_matches(&needle->ssl_config,
&check->ssl_config)) {
DEBUGF(infof(data,
"Connection #%ld has different SSL parameters, "
"can't reuse\n",
check->connection_id));
continue;
}
else if(check->ssl[FIRSTSOCKET].state != ssl_connection_complete) {
DEBUGF(infof(data,
"Connection #%ld has not started SSL connect, "
"can't reuse\n",
check->connection_id));
continue;
}
}
if((needle->handler->protocol & CURLPROTO_FTP) ||
((needle->handler->protocol & CURLPROTO_HTTP) && wantNTLM)) {
/* This is FTP or HTTP+NTLM, verify that we're using the same name
and password as well */
if(!strequal(needle->user, check->user) ||
!strequal(needle->passwd, check->passwd)) {
/* one of them was different */
continue;
}
credentialsMatch = TRUE;
}
match = TRUE;
}
}
else { /* The requested needle connection is using a proxy,
is the checked one using the same host, port and type? */
if(check->bits.proxy &&
(needle->proxytype == check->proxytype) &&
(needle->bits.tunnel_proxy == check->bits.tunnel_proxy) &&
Curl_raw_equal(needle->proxy.name, check->proxy.name) &&
needle->port == check->port) {
/* This is the same proxy connection, use it! */
match = TRUE;
}
}
}
if(match) {
chosen = check;
if(match) {
chosen = check;
/* If we are not looking for an NTLM connection, we can choose this one
immediately. */
if(!wantNTLM)
break;
/* If we are not looking for an NTLM connection, we can choose this one
immediately. */
if(!wantNTLM)
break;
/* Otherwise, check if this is already authenticating with the right
credentials. If not, keep looking so that we can reuse NTLM
connections if possible. (Especially we must reuse the same
connection if partway through a handshake!) */
if(credentialsMatch && chosen->ntlm.state != NTLMSTATE_NONE)
break;
/* Otherwise, check if this is already authenticating with the right
credentials. If not, keep looking so that we can reuse NTLM
connections if possible. (Especially we must reuse the same
connection if partway through a handshake!) */
if(credentialsMatch && chosen->ntlm.state != NTLMSTATE_NONE)
break;
}
}
}
@ -3170,53 +2983,67 @@ ConnectionExists(struct SessionHandle *data,
return FALSE; /* no matching connecting exists */
}
/*
* This function kills and removes an existing connection in the connection
* cache. The connection that has been unused for the longest time.
*
* Returns -1 if it can't find any unused connection to kill.
* Returns FALSE if it can't find any unused connection to kill.
*/
static long
static bool
ConnectionKillOne(struct SessionHandle *data)
{
long i;
struct connectdata *conn;
struct conncache *bc = data->state.conn_cache;
struct curl_hash_iterator iter;
struct curl_llist_element *curr;
struct curl_hash_element *he;
long highscore=-1;
long connindex=-1;
long score;
struct timeval now;
struct connectdata *conn_candidate = NULL;
struct connectbundle *bundle;
now = Curl_tvnow();
for(i=0; data->state.connc && (i< data->state.connc->num); i++) {
conn = data->state.connc->connects[i];
Curl_hash_start_iterate(bc->hash, &iter);
if(!conn || conn->inuse)
continue;
he = Curl_hash_next_element(&iter);
while(he) {
struct connectdata *conn;
/* Set higher score for the age passed since the connection was used */
score = Curl_tvdiff(now, conn->now);
bundle = he->ptr;
if(score > highscore) {
highscore = score;
connindex = i;
curr = bundle->conn_list->head;
while(curr) {
conn = curr->ptr;
if(!conn->inuse) {
/* Set higher score for the age passed since the connection was used */
score = Curl_tvdiff(now, conn->now);
if(score > highscore) {
highscore = score;
conn_candidate = conn;
}
}
curr = curr->next;
}
he = Curl_hash_next_element(&iter);
}
if(connindex >= 0) {
if(conn_candidate) {
/* Set the connection's owner correctly */
conn = data->state.connc->connects[connindex];
conn->data = data;
conn_candidate->data = data;
bundle = conn_candidate->bundle;
/* the winner gets the honour of being disconnected */
(void)Curl_disconnect(conn, /* dead_connection */ FALSE);
(void)Curl_disconnect(conn_candidate, /* dead_connection */ FALSE);
/* clean the array entry */
data->state.connc->connects[connindex] = NULL;
return TRUE;
}
return connindex; /* return the available index or -1 */
return FALSE;
}
/* this connection can now be marked 'idle' */
@ -3234,41 +3061,16 @@ ConnectionDone(struct connectdata *conn)
* The given connection should be unique. That must've been checked prior to
* this call.
*/
static void ConnectionStore(struct SessionHandle *data,
struct connectdata *conn)
static CURLcode ConnectionStore(struct SessionHandle *data,
struct connectdata *conn)
{
long i;
for(i=0; i< data->state.connc->num; i++) {
if(!data->state.connc->connects[i])
break;
}
if(i == data->state.connc->num) {
/* there was no room available, kill one */
i = ConnectionKillOne(data);
if(-1 != i)
infof(data, "Connection (#%ld) was killed to make room (holds %ld)\n",
i, data->state.connc->num);
else
infof(data, "This connection did not fit in the connection cache\n");
}
static int connection_id_counter = 0;
conn->connectindex = i; /* Make the child know where the pointer to this
particular data is stored. But note that this -1
if this is not within the cache and this is
probably not checked for everywhere (yet). */
conn->inuse = TRUE;
if(-1 != i) {
/* Only do this if a true index was returned, if -1 was returned there
is no room in the cache for an unknown reason and we cannot store
this there.
/* Assign a number to the connection for easier tracking in the log
output */
conn->connection_id = connection_id_counter++;
TODO: make sure we really can work with more handles than positions in
the cache, or possibly we should (allow to automatically) resize the
connection cache when we add more easy handles to a multi handle!
*/
data->state.connc->connects[i] = conn; /* fill in this */
conn->data = data;
}
return Curl_conncache_add_conn(data->state.conn_cache, conn);
}
/* after a TCP connection to the proxy has been verified, this function does
@ -3318,7 +3120,7 @@ static CURLcode ConnectPlease(struct SessionHandle *data,
infof(data, "About to connect() to %s%s port %ld (#%ld)\n",
conn->bits.proxy?"proxy ":"",
hostname, conn->port, conn->connectindex);
hostname, conn->port, conn->connection_id);
#else
(void)data;
#endif
@ -3359,7 +3161,7 @@ void Curl_verboseconnect(struct connectdata *conn)
if(conn->data->set.verbose)
infof(conn->data, "Connected to %s (%s) port %ld (#%ld)\n",
conn->bits.proxy ? conn->proxy.dispname : conn->host.dispname,
conn->ip_addr_str, conn->port, conn->connectindex);
conn->ip_addr_str, conn->port, conn->connection_id);
}
#endif
@ -3618,7 +3420,7 @@ static struct connectdata *allocate_conn(struct SessionHandle *data)
conn->sock[FIRSTSOCKET] = CURL_SOCKET_BAD; /* no file descriptor */
conn->sock[SECONDARYSOCKET] = CURL_SOCKET_BAD; /* no file descriptor */
conn->connectindex = -1; /* no index */
conn->connection_id = -1; /* no ID */
conn->port = -1; /* unknown at this point */
/* Default protocol-independent behavior doesn't support persistent
@ -4898,7 +4700,7 @@ static CURLcode create_conn(struct SessionHandle *data,
urllen=LEAST_PATH_ALLOC;
/*
* We malloc() the buffers below urllen+2 to make room for to possibilities:
* We malloc() the buffers below urllen+2 to make room for 2 possibilities:
* 1 - an extra terminating zero
* 2 - an extra slash (in case a syntax like "www.host.com?moo" is used)
*/
@ -4960,8 +4762,8 @@ static CURLcode create_conn(struct SessionHandle *data,
/* according to rfc3986, allow the query (?foo=bar)
also on protocols that can't handle it.
cut the string-part after '?'
*/
cut the string-part after '?'
*/
/* terminate the string */
path_q_sep[0] = 0;
@ -4975,7 +4777,7 @@ static CURLcode create_conn(struct SessionHandle *data,
if(conn->bits.proxy_user_passwd) {
result = parse_proxy_auth(data, conn);
if(result != CURLE_OK)
return result;
return result;
}
/*************************************************************
@ -5178,7 +4980,7 @@ static CURLcode create_conn(struct SessionHandle *data,
fix_hostname(data, conn, &conn->host);
infof(data, "Re-using existing connection! (#%ld) with host %s\n",
conn->connectindex,
conn->connection_id,
conn->proxy.name?conn->proxy.dispname:conn->host.dispname);
}
else {
@ -5443,12 +5245,8 @@ CURLcode Curl_done(struct connectdata **connp,
state it is for re-using, so we're forced to close it. In a perfect world
we can add code that keep track of if we really must close it here or not,
but currently we have no such detail knowledge.
connectindex == -1 here means that the connection has no spot in the
connection cache and thus we must disconnect it here.
*/
if(data->set.reuse_forbid || conn->bits.close || premature ||
(-1 == conn->connectindex)) {
if(data->set.reuse_forbid || conn->bits.close || premature) {
CURLcode res2 = Curl_disconnect(conn, premature); /* close connection */
/* If we had an error already, make sure we return that one. But
@ -5460,10 +5258,10 @@ CURLcode Curl_done(struct connectdata **connp,
ConnectionDone(conn); /* the connection is no longer in use */
/* remember the most recently used connection */
data->state.lastconnect = conn->connectindex;
data->state.lastconnect = conn;
infof(data, "Connection #%ld to host %s left intact\n",
conn->connectindex,
conn->connection_id,
conn->bits.httpproxy?conn->proxy.dispname:conn->host.dispname);
}

9
lib/url.h
View File

@ -46,15 +46,6 @@ CURLcode Curl_protocol_doing(struct connectdata *conn, bool *done);
CURLcode Curl_setup_conn(struct connectdata *conn,
bool *protocol_done);
/* create a connection cache */
struct conncache *Curl_mk_connc(int type, long amount);
/* free a connection cache */
void Curl_rm_connc(struct conncache *c);
/* Change number of entries of a connection cache */
CURLcode Curl_ch_connc(struct SessionHandle *data,
struct conncache *c,
long newamount);
int Curl_protocol_getsock(struct connectdata *conn,
curl_socket_t *socks,
int numsocks);

43
lib/urldata.h
View File

@ -795,8 +795,8 @@ struct connectdata {
consideration (== only for pipelining). */
/**** Fields set when inited and not modified again */
long connectindex; /* what index in the connection cache connects index this
particular struct has */
long connection_id; /* Contains a unique number to make it easier to
track the connections in the log output */
/* 'dns_entry' is the particular host we use. This points to an entry in the
DNS cache and it will not get pruned while locked. It gets unlocked in
@ -924,7 +924,6 @@ struct connectdata {
handle */
bool server_supports_pipelining; /* TRUE if server supports pipelining,
set after first response */
struct curl_llist *send_pipe; /* List of handles waiting to
send on this pipeline */
struct curl_llist *recv_pipe; /* List of handles waiting to read
@ -934,7 +933,6 @@ struct connectdata {
struct curl_llist *done_pipe; /* Handles that are finished, but
still reference this connectdata */
#define MAX_PIPELINE_LENGTH 5
char* master_buffer; /* The master buffer allocated on-demand;
used for pipelining. */
size_t read_pos; /* Current read position in the master buffer */
@ -1011,6 +1009,8 @@ struct connectdata {
TUNNEL_CONNECT, /* CONNECT has been sent off */
TUNNEL_COMPLETE /* CONNECT response received completely */
} tunnel_state[2]; /* two separate ones to allow FTP */
struct connectbundle *bundle; /* The bundle we are member of */
};
/* The end of connectdata. */
@ -1146,18 +1146,6 @@ struct auth {
be RFC compliant */
};
struct conncache {
/* 'connects' will be an allocated array with pointers. If the pointer is
set, it holds an allocated connection. */
struct connectdata **connects;
long num; /* number of entries of the 'connects' array */
enum {
CONNCACHE_PRIVATE, /* used for an easy handle alone */
CONNCACHE_MULTI /* shared within a multi handle */
} type;
};
struct UrlState {
enum {
Curl_if_none,
@ -1165,13 +1153,20 @@ struct UrlState {
Curl_if_multi
} used_interface;
struct conncache *connc; /* points to the connection cache this handle
uses */
/* Points to the connection cache */
struct conncache *conn_cache;
/* buffers to store authentication data in, as parsed from input options */
struct timeval keeps_speed; /* for the progress meter really */
long lastconnect; /* index of most recent connect or -1 if undefined */
struct connectdata *pending_conn; /* This points to the connection we want
to open when we are waiting in the
CONNECT_PEND state in the multi
interface. This to avoid recreating it
when we enter the CONNECT state again.
*/
struct connectdata *lastconnect; /* The last connection, NULL if undefined */
char *headerbuff; /* allocated buffer to store headers in */
size_t headersize; /* size of the allocation */
@ -1250,14 +1245,6 @@ struct UrlState {
/* for FTP downloads: how many CRLFs did we converted to LFs? */
curl_off_t crlf_conversions;
#endif
/* If set to non-NULL, there's a connection in a shared connection cache
that uses this handle so we can't kill this SessionHandle just yet but
must keep it around and add it to the list of handles to kill once all
its connections are gone */
void *shared_conn;
bool closed; /* set to TRUE when curl_easy_cleanup() has been called on this
handle, but it is kept around as mentioned for
shared_conn */
char *pathbuffer;/* allocated buffer to store the URL's path part in */
char *path; /* path to use, points to somewhere within the pathbuffer
area */
@ -1593,6 +1580,8 @@ struct UserDefined {
bool tcp_keepalive; /* use TCP keepalives */
long tcp_keepidle; /* seconds in idle before sending keepalive probe */
long tcp_keepintvl; /* seconds between TCP keepalive probes */
size_t maxconnects; /* Max idle connections in the connection cache */
};
struct Names {