Handle SSL_shutdown while in init more appropriately

Calling SSL_shutdown while in init previously gave a "1" response, meaning
everything was successfully closed down (even though it wasn't). Better is
to send our close_notify, but fail when trying to receive one.

The problem with doing a shutdown while in the middle of a handshake is
that once our close_notify is sent we shouldn't really do anything else
(including process handshake/CCS messages) until we've received a
close_notify back from the peer. However the peer might send a CCS before
acting on our close_notify - so we won't be able to read it because we're
not acting on CCS messages!

Reviewed-by: Viktor Dukhovni <viktor@openssl.org>

ssl/s3_lib.c

@@ -4340,6 +4340,21 @@ int ssl3_shutdown(SSL *s)
         }
 #endif
     } else if (!(s->shutdown & SSL_RECEIVED_SHUTDOWN)) {
+        if (SSL_in_init(s)) {
+            /*
+             * We can't shutdown properly if we are in the middle of a
+             * handshake. Doing so is problematic because the peer may send a
+             * CCS before it acts on our close_notify. However we should not
+             * continue to process received handshake messages or CCS once our
+             * close_notify has been sent. Therefore any close_notify from
+             * the peer will be unreadable because we have not moved to the next
+             * cipher state. Its best just to avoid this can-of-worms. Return
+             * an error if we are wanting to wait for a close_notify from the
+             * peer and we are in init.
+             */
+            SSLerr(SSL_F_SSL3_SHUTDOWN, SSL_R_SHUTDOWN_WHILE_IN_INIT);
+            return -1;
+        }
         /*
          * If we are waiting for a close from our peer, we are closed
          */

ssl/ssl.h

@@ -2713,6 +2713,7 @@ void ERR_load_SSL_strings(void);
 # define SSL_F_SSL3_SETUP_KEY_BLOCK                       157
 # define SSL_F_SSL3_SETUP_READ_BUFFER                     156
 # define SSL_F_SSL3_SETUP_WRITE_BUFFER                    291
+# define SSL_F_SSL3_SHUTDOWN                              396
 # define SSL_F_SSL3_WRITE_BYTES                           158
 # define SSL_F_SSL3_WRITE_PENDING                         159
 # define SSL_F_SSL_ADD_CERT_CHAIN                         318
@@ -3056,6 +3057,7 @@ void ERR_load_SSL_strings(void);
 # define SSL_R_SERVERHELLO_TLSEXT                         275
 # define SSL_R_SESSION_ID_CONTEXT_UNINITIALIZED           277
 # define SSL_R_SHORT_READ                                 219
+# define SSL_R_SHUTDOWN_WHILE_IN_INIT                     407
 # define SSL_R_SIGNATURE_ALGORITHMS_ERROR                 360
 # define SSL_R_SIGNATURE_FOR_NON_SIGNING_CERTIFICATE      220
 # define SSL_R_SRP_A_CALC                                 361

ssl/ssl_err.c

@@ -206,6 +206,7 @@ static ERR_STRING_DATA SSL_str_functs[] = {
     {ERR_FUNC(SSL_F_SSL3_SETUP_KEY_BLOCK), "ssl3_setup_key_block"},
     {ERR_FUNC(SSL_F_SSL3_SETUP_READ_BUFFER), "ssl3_setup_read_buffer"},
     {ERR_FUNC(SSL_F_SSL3_SETUP_WRITE_BUFFER), "ssl3_setup_write_buffer"},
+    {ERR_FUNC(SSL_F_SSL3_SHUTDOWN), "ssl3_shutdown"},
     {ERR_FUNC(SSL_F_SSL3_WRITE_BYTES), "ssl3_write_bytes"},
     {ERR_FUNC(SSL_F_SSL3_WRITE_PENDING), "ssl3_write_pending"},
     {ERR_FUNC(SSL_F_SSL_ADD_CERT_CHAIN), "ssl_add_cert_chain"},
@@ -647,6 +648,7 @@ static ERR_STRING_DATA SSL_str_reasons[] = {
     {ERR_REASON(SSL_R_SESSION_ID_CONTEXT_UNINITIALIZED),
      "session id context uninitialized"},
     {ERR_REASON(SSL_R_SHORT_READ), "short read"},
+    {ERR_REASON(SSL_R_SHUTDOWN_WHILE_IN_INIT), "shutdown while in init"},
     {ERR_REASON(SSL_R_SIGNATURE_ALGORITHMS_ERROR),
      "signature algorithms error"},
     {ERR_REASON(SSL_R_SIGNATURE_FOR_NON_SIGNING_CERTIFICATE),

ssl/ssl_lib.c

@@ -1060,10 +1060,7 @@ int SSL_shutdown(SSL *s)
         return -1;
     }
 
-    if ((s != NULL) && !SSL_in_init(s))
+    return s->method->ssl_shutdown(s);
-        return (s->method->ssl_shutdown(s));
-    else
-        return (1);
 }
 
 int SSL_renegotiate(SSL *s)