Fixes chunked transfer encoding in HTTP client API

ChangeLog

@@ -7,6 +7,17 @@ Version 1.6.11
 	- Null termination of strndup() implementation on systems missing it.
 	- Implementation of strnlen() on systems missing it.
 
+2011-01-14 Chandra Penke <chandrapenke(at)mcntech.com>
+
+	Fixes transfer encoding in the HTTP client API, which is currently
+	broken. The break was due to a regression caused by another
+	fix (tracker 3056713), which fixed an out of memory crash when
+	downloading large files. The previous fix changed the
+	http_ReadHttpGet() implementation so that data already read by the
+	user was discarded. However, it only worked for transfers where
+	the content length was specified. This fix extends the previous
+	implementation to cover chunked transfer encoding.
+
 2011-01-14 Chandra Penke <chandrapenke(at)mcntech.com>
 
 	Minor change in membuffer.c to include "membuffer.h" without looking

upnp/src/genlib/net/http/httpparser.c

@@ -1569,7 +1569,7 @@ parser_parse_entity_using_clen( INOUT http_parser_t * parser )
     /* determine entity (i.e. body) length so far */
     parser->msg.entity.length =
         parser->msg.msg.length - parser->entity_start_position + 
-        parser->msg.entity_offset;
+        parser->msg.amount_discarded;
 
     if( parser->msg.entity.length < parser->content_length ) {
         /* more data to be read */
@@ -1577,16 +1577,16 @@ parser_parse_entity_using_clen( INOUT http_parser_t * parser )
     } else {
         if( parser->msg.entity.length > parser->content_length ) {
             /* silently discard extra data */
-            parser->msg.msg.buf[parser->entity_start_position -
+            parser->msg.msg.buf[parser->entity_start_position +
-                                parser->msg.entity_offset +
+                                parser->content_length -
-                                parser->content_length] = '\0';
+                                parser->msg.amount_discarded] = '\0';
         }
         /* save entity length */
         parser->msg.entity.length = parser->content_length;
 
         /* save entity start ptr; (the very last thing to do) */
         parser->msg.entity.buf = parser->msg.msg.buf +
-            parser->entity_start_position;
+                                 parser->entity_start_position;
 
         /* done reading entity */
         parser->position = POS_COMPLETE;
@@ -1719,7 +1719,7 @@ parser_parse_chunky_entity( INOUT http_parser_t * parser )
     if( parser->chunk_size == 0 ) {
         /* done reading entity; determine length of entity */
         parser->msg.entity.length = parser->scanner.cursor -
-            parser->entity_start_position;
+            parser->entity_start_position + parser->msg.amount_discarded;
 
         /* read entity headers */
         parser->ent_position = ENTREAD_CHUNKY_HEADERS;
@@ -1753,7 +1753,8 @@ parser_parse_entity_until_close( INOUT http_parser_t * parser )
     cursor = parser->msg.msg.length;
 
     /* update entity length */
-    parser->msg.entity.length = cursor - parser->entity_start_position;
+    parser->msg.entity.length = cursor - parser->entity_start_position +
+                                parser->msg.amount_discarded;
 
     /* update pointer */
     parser->msg.entity.buf =
@@ -1958,7 +1959,7 @@ parser_response_init( OUT http_parser_t * parser,
     parser_init( parser );
     parser->msg.is_request = FALSE;
     parser->msg.request_method = request_method;
-    parser->msg.entity_offset = 0;
+    parser->msg.amount_discarded = 0;
     parser->position = POS_RESPONSE_LINE;
 }
 

upnp/src/genlib/net/http/httpreadwrite.c

@@ -1242,8 +1242,8 @@ int http_ReadHttpGet(
 		return UPNP_E_BAD_RESPONSE;
 	}
 	/* read more if necessary entity */
-	while (handle->response.msg.entity_offset + *size >
+	while (handle->response.msg.amount_discarded + *size >
-		handle->response.msg.entity.length &&
+	       handle->response.msg.entity.length &&
 	       !handle->cancel &&
 	       handle->response.position != POS_COMPLETE) {
 		num_read = sock_read(&handle->sock_info, tempbuf,
@@ -1287,20 +1287,25 @@ int http_ReadHttpGet(
 			return num_read;
 		}
 	}
-	if (handle->response.msg.entity_offset + *size >
-	    handle->response.msg.entity.length)
-		*size = handle->response.msg.entity.length -
-		    handle->response.msg.entity_offset;
-	memcpy(buf, &handle->response.msg.msg.buf[handle->response.entity_start_position],
-		*size);
-	/* FIXME: testing size AFTER memcopy? Weird... */
-	if (*size > 0)
-		membuffer_delete(&handle->response.msg.msg,
-			handle->response.entity_start_position, *size);
-	handle->response.msg.entity_offset += *size;
 	if (handle->cancel) {
 		return UPNP_E_CANCELED;
 	}
+	/* truncate size to fall within available data */
+	if (handle->response.msg.amount_discarded + *size >
+	    handle->response.msg.entity.length)
+		*size = handle->response.msg.entity.length -
+			handle->response.msg.amount_discarded;
+	/* copy data to user buffer. delete copied data */
+	if (*size > 0) {
+		memcpy(buf, &handle->response.msg.msg.buf[handle->response.entity_start_position],
+			*size);
+		membuffer_delete(&handle->response.msg.msg,
+			handle->response.entity_start_position, *size);
+		/* update scanner position. needed for chunked transfers */
+		handle->response.scanner.cursor -= *size;
+		/* update amount discarded */
+		handle->response.msg.amount_discarded += *size;
+	}
 
 	return UPNP_E_SUCCESS;
 }

upnp/src/inc/httpparser.h

@@ -183,6 +183,10 @@ typedef struct {
 	int status_code;
 	/*! response only. */
 	membuffer status_msg;
+       /*! response only. the amount of data that's been read by the user, that's no
+        *  longer in the raw message buffer.
+        */
+       size_t amount_discarded;
 	/* fields used in both request or response messages. */
 	/*! if TRUE, msg is a request, else response. */
 	int is_request;
@@ -199,8 +203,6 @@ typedef struct {
 	membuffer msg;
         /*! storage for url string. */
         char *urlbuf;
-	/*! . */
-        size_t entity_offset;
 } http_message_t;
 
 typedef struct {
@@ -216,6 +218,8 @@ typedef struct {
 	int ent_position;
 	unsigned int content_length;
 	int chunk_size;
+       /*! offset in the the raw message buffer, which contains the message body.
+        *  preceding this are the headers of the messsage. */
 	size_t entity_start_position;
 	scanner_t scanner;
 } http_parser_t;