Lazily initialise the compression buffer

With read pipelining we use multiple SSL3_RECORD structures for reading.
There are SSL_MAX_PIPELINES (32) of them defined (typically not all of these
would be used). Each one has a 16k compression buffer allocated! This
results in a significant amount of memory being consumed which, most of the
time, is not needed.  This change swaps the allocation of the compression
buffer to be lazy so that it is only done immediately before it is actually
used.

Reviewed-by: Tim Hudson <tjh@openssl.org>

ssl/record/rec_layer_s3.c

@@ -223,11 +223,6 @@ void RECORD_LAYER_reset_write_sequence(RECORD_LAYER *rl)
     memset(rl->write_sequence, 0, sizeof(rl->write_sequence));
 }
 
-int RECORD_LAYER_setup_comp_buffer(RECORD_LAYER *rl)
-{
-    return SSL3_RECORD_setup((rl)->rrec, SSL_MAX_PIPELINES);
-}
-
 int ssl3_pending(const SSL *s)
 {
     unsigned int i;

ssl/record/record.h

@@ -325,7 +325,6 @@ int RECORD_LAYER_write_pending(RECORD_LAYER *rl);
 int RECORD_LAYER_set_data(RECORD_LAYER *rl, const unsigned char *buf, int len);
 void RECORD_LAYER_reset_read_sequence(RECORD_LAYER *rl);
 void RECORD_LAYER_reset_write_sequence(RECORD_LAYER *rl);
-int RECORD_LAYER_setup_comp_buffer(RECORD_LAYER *rl);
 int RECORD_LAYER_is_sslv2_record(RECORD_LAYER *rl);
 unsigned int RECORD_LAYER_get_rrec_length(RECORD_LAYER *rl);
 __owur int ssl3_pending(const SSL *s);

ssl/record/record_locl.h

@@ -193,7 +193,6 @@ int ssl3_release_write_buffer(SSL *s);
 
 void SSL3_RECORD_clear(SSL3_RECORD *r, unsigned int num_recs);
 void SSL3_RECORD_release(SSL3_RECORD *r, unsigned int num_recs);
-int SSL3_RECORD_setup(SSL3_RECORD *r, unsigned int num_recs);
 void SSL3_RECORD_set_seq_num(SSL3_RECORD *r, const unsigned char *seq_num);
 int ssl3_get_record(SSL *s);
 __owur int ssl3_do_compress(SSL *ssl, SSL3_RECORD *wr);

ssl/record/ssl3_record.c

@@ -157,24 +157,6 @@ void SSL3_RECORD_release(SSL3_RECORD *r, unsigned int num_recs)
     }
 }
 
-int SSL3_RECORD_setup(SSL3_RECORD *r, unsigned int num_recs)
-{
-    unsigned int i;
-
-    for (i = 0; i < num_recs; i++) {
-        if (r[i].comp == NULL)
-            r[i].comp = (unsigned char *)
-                OPENSSL_malloc(SSL3_RT_MAX_ENCRYPTED_LENGTH);
-        if (r[i].comp == NULL) {
-            if (i > 0)
-                SSL3_RECORD_release(r, i);
-            return 0;
-        }
-    }
-
-    return 1;
-}
-
 void SSL3_RECORD_set_seq_num(SSL3_RECORD *r, const unsigned char *seq_num)
 {
     memcpy(r->seq_num, seq_num, SEQ_NUM_SIZE);
@@ -626,16 +608,23 @@ int ssl3_do_uncompress(SSL *ssl, SSL3_RECORD *rr)
 #ifndef OPENSSL_NO_COMP
     int i;
 
+    if (rr->comp == NULL) {
+        rr->comp = (unsigned char *)
+            OPENSSL_malloc(SSL3_RT_MAX_ENCRYPTED_LENGTH);
+    }
+    if (rr->comp == NULL)
+        return 0;
+
     i = COMP_expand_block(ssl->expand, rr->comp,
                           SSL3_RT_MAX_PLAIN_LENGTH, rr->data,
                           (int)rr->length);
     if (i < 0)
-        return (0);
+        return 0;
     else
         rr->length = i;
     rr->data = rr->comp;
 #endif
-    return (1);
+    return 1;
 }
 
 int ssl3_do_compress(SSL *ssl, SSL3_RECORD *wr)

ssl/s3_enc.c

@@ -251,8 +251,6 @@ int ssl3_change_cipher_state(SSL *s, int which)
                        SSL_R_COMPRESSION_LIBRARY_ERROR);
                 goto err2;
             }
-            if (!RECORD_LAYER_setup_comp_buffer(&s->rlayer))
-                goto err;
         }
 #endif
         RECORD_LAYER_reset_read_sequence(&s->rlayer);

ssl/t1_enc.c

@@ -260,8 +260,6 @@ int tls1_change_cipher_state(SSL *s, int which)
                        SSL_R_COMPRESSION_LIBRARY_ERROR);
                 goto err2;
             }
-            if (!RECORD_LAYER_setup_comp_buffer(&s->rlayer))
-                goto err;
         }
 #endif
         /*