Add initial state machine rewrite code

This is the first drop of the new state machine code.

The rewrite has the following objectives:
- Remove duplication of state code between client and server
- Remove duplication of state code between TLS and DTLS
- Simplify transitions and bring the logic together in a single location
  so that it is easier to validate
- Remove duplication of code between each of the message handling functions
- Receive a message first and then work out whether that is a valid
  transition - not the other way around (the other way causes lots of issues
  where we are expecting one type of message next but actually get something
  else)
- Separate message flow state from handshake state (in order to better
  understand each)
  - message flow state = when to flush buffers; handling restarts in the
    event of NBIO events; handling the common flow of steps for reading a
    message and the common flow of steps for writing a message etc
  - handshake state = what handshake message are we working on now
- Control complexity: only the state machine can change state: keep all
  the state changes local to a file

This builds on previous state machine related work:
- Surface CCS processing in the state machine
- Version negotiation rewrite

Reviewed-by: Tim Hudson <tjh@openssl.org>
Reviewed-by: Richard Levitte <levitte@openssl.org>

ssl/ssl_locl.h

@@ -717,6 +717,137 @@ struct ssl_comp_st {
 DECLARE_STACK_OF(SSL_COMP)
 DECLARE_LHASH_OF(SSL_SESSION);
 
+/*
+ * The valid handshake states (one for each type message sent and one for each
+ * type of message received). There are also two "special" states:
+ * TLS = TLS or DTLS state
+ * DTLS = DTLS specific state
+ * CR/SR = Client Read/Server Read
+ * CW/SW = Client Write/Server Write
+ *
+ * The "special" states are:
+ * TLS_ST_BEFORE = No handshake has been initiated yet
+ * TLS_ST_OK = A handshake has been successfully completed
+ */
+enum HANDSHAKE_STATE {
+    TLS_ST_BEFORE,
+    TLS_ST_OK,
+    DTLS_ST_CR_HELLO_VERIFY_REQUEST,
+    TLS_ST_CR_SRVR_HELLO,
+    TLS_ST_CR_CERT,
+    TLS_ST_CR_CERT_STATUS,
+    TLS_ST_CR_KEY_EXCH,
+    TLS_ST_CR_CERT_REQ,
+    TLS_ST_CR_SRVR_DONE,
+    TLS_ST_CR_SESSION_TICKET,
+    TLS_ST_CR_CHANGE,
+    TLS_ST_CR_FINISHED,
+    TLS_ST_CW_CLNT_HELLO,
+    TLS_ST_CW_CERT,
+    TLS_ST_CW_KEY_EXCH,
+    TLS_ST_CW_CERT_VRFY,
+    TLS_ST_CW_CHANGE,
+    TLS_ST_CW_NEXT_PROTO,
+    TLS_ST_CW_FINISHED,
+    TLS_ST_SW_HELLO_REQ,
+    TLS_ST_SR_CLNT_HELLO,
+    DTLS_ST_SW_HELLO_VERIFY_REQUEST,
+    TLS_ST_SW_SRVR_HELLO,
+    TLS_ST_SW_CERT,
+    TLS_ST_SW_KEY_EXCH,
+    TLS_ST_SW_CERT_REQ,
+    TLS_ST_SW_SRVR_DONE,
+    TLS_ST_SR_CERT,
+    TLS_ST_SR_KEY_EXCH,
+    TLS_ST_SR_CERT_VRFY,
+    TLS_ST_SR_NEXT_PROTO,
+    TLS_ST_SR_CHANGE,
+    TLS_ST_SR_FINISHED,
+    TLS_ST_SW_SESSION_TICKET,
+    TLS_ST_SW_CERT_STATUS,
+    TLS_ST_SW_CHANGE,
+    TLS_ST_SW_FINISHED
+};
+
+/*
+ * Valid return codes used for functions performing work prior to or after
+ * sending or receiving a message
+ */
+enum WORK_STATE {
+    /* Something went wrong */
+    WORK_ERROR,
+    /* We're done working and there shouldn't be anything else to do after */
+    WORK_FINISHED_STOP,
+    /* We're done working move onto the next thing */
+    WORK_FINISHED_CONTINUE,
+    /* We're working on phase A */
+    WORK_MORE_A,
+    /* We're working on phase B */
+    WORK_MORE_B
+};
+
+/* Write transition return codes */
+enum WRITE_TRAN {
+    /* Something went wrong */
+    WRITE_TRAN_ERROR,
+    /* A transition was successfully completed and we should continue */
+    WRITE_TRAN_CONTINUE,
+    /* There is no more write work to be done */
+    WRITE_TRAN_FINISHED
+};
+
+/* Message processing return codes */
+enum MSG_PROCESS_RETURN {
+    MSG_PROCESS_ERROR,
+    MSG_PROCESS_FINISHED_READING,
+    MSG_PROCESS_CONTINUE_PROCESSING,
+    MSG_PROCESS_CONTINUE_READING
+};
+
+/* Message flow states */
+enum MSG_FLOW_STATE {
+    /* No handshake in progress */
+    MSG_FLOW_UNINITED,
+    /* A permanent error with this connection */
+    MSG_FLOW_ERROR,
+    /* We are about to renegotiate */
+    MSG_FLOW_RENEGOTIATE,
+    /* We are reading messages */
+    MSG_FLOW_READING,
+    /* We are writing messages */
+    MSG_FLOW_WRITING,
+    /* Handshake has finished */
+    MSG_FLOW_FINISHED
+};
+
+/* Read states */
+enum READ_STATE {
+    READ_STATE_HEADER,
+    READ_STATE_BODY,
+    READ_STATE_POST_PROCESS
+};
+
+/* Write states */
+enum WRITE_STATE {
+    WRITE_STATE_TRANSITION,
+    WRITE_STATE_PRE_WORK,
+    WRITE_STATE_SEND,
+    WRITE_STATE_POST_WORK
+};
+
+struct statem_st {
+    enum MSG_FLOW_STATE state;
+    enum WRITE_STATE write_state;
+    enum WORK_STATE write_state_work;
+    enum READ_STATE read_state;
+    enum WORK_STATE read_state_work;
+    enum HANDSHAKE_STATE hand_state;
+    int read_state_first_init;
+    int use_timer;
+};
+typedef struct statem_st STATEM;
+
+
 struct ssl_ctx_st {
     const SSL_METHOD *method;
     STACK_OF(SSL_CIPHER) *cipher_list;
@@ -1012,6 +1143,8 @@ struct ssl_st {
     int shutdown;
     /* where we are */
     int state;
+    STATEM statem;
+
     BUF_MEM *init_buf;          /* buffer used during init */
     void *init_msg;             /* pointer to handshake message body, set by
                                  * ssl3_get_message() */
@@ -1951,6 +2084,10 @@ __owur int ssl3_new(SSL *s);
 void ssl3_free(SSL *s);
 __owur int ssl3_accept(SSL *s);
 __owur int ssl3_connect(SSL *s);
+void statem_clear(SSL *s);
+void statem_set_renegotiate(SSL *s);
+void statem_set_error(SSL *s);
+__owur int statem_client_app_data_allowed(SSL *s);
 __owur int ssl3_read(SSL *s, void *buf, int len);
 __owur int ssl3_peek(SSL *s, void *buf, int len);
 __owur int ssl3_write(SSL *s, const void *buf, int len);