Convert __thread to pthreads for Thread Local Storage

In theory the pthreads approach for Thread Local Storage should be more
portable.

This also changes some APIs in order to accommodate this change. In
particular ASYNC_init_pool is renamed ASYNC_init_thread and
ASYNC_free_pool is renamed ASYNC_cleanup_thread. Also introduced ASYNC_init
and ASYNC_cleanup.

Reviewed-by: Rich Salz <rsalz@openssl.org>

apps/s_client.c

@@ -1208,7 +1208,7 @@ int s_client_main(int argc, char **argv)
 
     if (async) {
         SSL_CTX_set_mode(ctx, SSL_MODE_ASYNC);
-        ASYNC_init_pool(0, 0);
+        ASYNC_init(1, 0, 0);
     }
 
     if (!config_ctx(cctx, ssl_args, ctx, 1, jpake_secret == NULL))
@@ -2102,7 +2102,7 @@ int s_client_main(int argc, char **argv)
         SSL_free(con);
     }
     if (async) {
-        ASYNC_free_pool();
+        ASYNC_cleanup(1);
     }
 #if !defined(OPENSSL_NO_NEXTPROTONEG)
     OPENSSL_free(next_proto.data);

apps/s_server.c

@@ -1660,7 +1660,7 @@ int s_server_main(int argc, char *argv[])
 
     if (async) {
         SSL_CTX_set_mode(ctx, SSL_MODE_ASYNC);
-        ASYNC_init_pool(0, 0);
+        ASYNC_init(1, 0, 0);
     }
 
 #ifndef OPENSSL_NO_SRTP
@@ -1974,7 +1974,7 @@ int s_server_main(int argc, char *argv[])
     BIO_free(bio_s_msg);
     bio_s_msg = NULL;
     if (async) {
-        ASYNC_free_pool();
+        ASYNC_cleanup(1);
     }
     return (ret);
 }

crypto/async/arch/async_null.c

@@ -61,6 +61,11 @@ int async_pipe(OSSL_ASYNC_FD *pipefds)
     return -1;
 }
 
+int async_close_fd(OSSL_ASYNC_FD fd)
+{
+    return 0;
+}
+
 int async_write1(OSSL_ASYNC_FD fd, const void *buf)
 {
     return -1;
@@ -71,5 +76,10 @@ int async_read1(OSSL_ASYNC_FD fd, void *buf)
     return -1;
 }
 
+int async_thread_local_init(void)
+{
+    return 0;
+}
+
 #endif
 

crypto/async/arch/async_posix.c

@@ -61,11 +61,20 @@
 # include <openssl/crypto.h>
 # include <openssl/async.h>
 
-__thread async_ctx *posixctx;
+pthread_key_t posixctx;
-__thread async_pool *posixpool;
+pthread_key_t posixpool;
 
 #define STACKSIZE       32768
 
+int async_thread_local_init(void)
+{
+    if (pthread_key_create(&posixctx, NULL) != 0
+            || pthread_key_create(&posixpool, NULL) != 0)
+        return 0;
+
+    return 1;
+}
+
 int async_fibre_init(async_fibre *fibre)
 {
     void *stack = NULL;

crypto/async/arch/async_posix.h

@@ -52,12 +52,14 @@
  */
 #include <openssl/e_os2.h>
 
-#ifdef OPENSSL_SYS_UNIX
+#if defined(OPENSSL_SYS_UNIX) && defined(OPENSSL_THREADS)
 
 # include <unistd.h>
 
 # if _POSIX_VERSION >= 200112L
 
+# include <pthread.h>
+
 #  define ASYNC_POSIX
 #  define ASYNC_ARCH
 
@@ -73,8 +75,8 @@
 #  include <setjmp.h>
 #  include "e_os.h"
 
-extern __thread async_ctx *posixctx;
+extern pthread_key_t posixctx;
-extern __thread async_pool *posixpool;
+extern pthread_key_t posixpool;
 
 typedef struct async_fibre_st {
     ucontext_t fibre;
@@ -82,10 +84,10 @@ typedef struct async_fibre_st {
     int env_init;
 } async_fibre;
 
-#  define async_set_ctx(nctx)             (posixctx = (nctx))
+#  define async_set_ctx(nctx)  (pthread_setspecific(posixctx , (nctx)) == 0)
-#  define async_get_ctx()                 (posixctx)
+#  define async_get_ctx()      ((async_ctx *)pthread_getspecific(posixctx))
-#  define async_set_pool(p)               (posixpool = (p))
+#  define async_set_pool(p)    (pthread_setspecific(posixpool , (p)) == 0)
-#  define async_get_pool()                (posixpool)
+#  define async_get_pool()     ((async_pool *)pthread_getspecific(posixpool))
 
 static inline int async_fibre_swapcontext(async_fibre *o, async_fibre *n, int r)
 {

crypto/async/async.c

@@ -156,7 +156,7 @@ static ASYNC_JOB *async_get_pool_job(void) {
          * Pool has not been initialised, so init with the defaults, i.e.
          * no max size and no pre-created jobs
          */
-        if (ASYNC_init_pool(0, 0) == 0)
+        if (ASYNC_init_thread(0, 0) == 0)
             return NULL;
         pool = async_get_pool();
     }
@@ -328,30 +328,36 @@ static void async_empty_pool(async_pool *pool)
     } while (job);
 }
 
-int ASYNC_init_pool(size_t max_size, size_t init_size)
+int ASYNC_init(int init_thread, size_t max_size, size_t init_size)
+{
+    if (!async_thread_local_init())
+        return 0;
+
+    if (init_thread)
+        return ASYNC_init_thread(max_size, init_size);
+
+    return 1;
+}
+
+int ASYNC_init_thread(size_t max_size, size_t init_size)
 {
     async_pool *pool;
     size_t curr_size = 0;
 
-    if (init_size > max_size || max_size == 0) {
+    if (init_size > max_size) {
-        ASYNCerr(ASYNC_F_ASYNC_INIT_POOL, ASYNC_R_INVALID_POOL_SIZE);
+        ASYNCerr(ASYNC_F_ASYNC_INIT_THREAD, ASYNC_R_INVALID_POOL_SIZE);
-        return 0;
-    }
-
-    if(async_get_pool() != NULL) {
-        ASYNCerr(ASYNC_F_ASYNC_INIT_POOL, ASYNC_R_POOL_ALREADY_INITED);
         return 0;
     }
 
     pool = OPENSSL_zalloc(sizeof *pool);
     if (pool == NULL) {
-        ASYNCerr(ASYNC_F_ASYNC_INIT_POOL, ERR_R_MALLOC_FAILURE);
+        ASYNCerr(ASYNC_F_ASYNC_INIT_THREAD, ERR_R_MALLOC_FAILURE);
         return 0;
     }
 
     pool->jobs = sk_ASYNC_JOB_new_null();
     if (pool->jobs == NULL) {
-        ASYNCerr(ASYNC_F_ASYNC_INIT_POOL, ERR_R_MALLOC_FAILURE);
+        ASYNCerr(ASYNC_F_ASYNC_INIT_THREAD, ERR_R_MALLOC_FAILURE);
         OPENSSL_free(pool);
         return 0;
     }
@@ -379,7 +385,7 @@ int ASYNC_init_pool(size_t max_size, size_t init_size)
     pool->curr_size = curr_size;
 
     if (!async_set_pool(pool)) {
-        ASYNCerr(ASYNC_F_ASYNC_INIT_POOL, ASYNC_R_FAILED_TO_SET_POOL);
+        ASYNCerr(ASYNC_F_ASYNC_INIT_THREAD, ASYNC_R_FAILED_TO_SET_POOL);
         goto err;
     }
 
@@ -397,15 +403,25 @@ static void async_free_pool_internal(async_pool *pool)
     async_empty_pool(pool);
     sk_ASYNC_JOB_free(pool->jobs);
     OPENSSL_free(pool);
-    async_set_pool(NULL);
+    (void)async_set_pool(NULL);
     async_ctx_free();
 }
 
-void ASYNC_free_pool(void)
+void ASYNC_cleanup_thread(void)
 {
     async_free_pool_internal(async_get_pool());
 }
 
+void ASYNC_cleanup(int cleanupthread)
+{
+    /*
+     * We don't actually have any global cleanup at the moment so just cleanup
+     * the thread
+     */
+    if (cleanupthread)
+        ASYNC_cleanup_thread();
+}
+
 ASYNC_JOB *ASYNC_get_current_job(void)
 {
     async_ctx *ctx;

crypto/async/async_err.c

@@ -71,7 +71,7 @@
 
 static ERR_STRING_DATA ASYNC_str_functs[] = {
     {ERR_FUNC(ASYNC_F_ASYNC_CTX_NEW), "async_ctx_new"},
-    {ERR_FUNC(ASYNC_F_ASYNC_INIT_POOL), "ASYNC_init_pool"},
+    {ERR_FUNC(ASYNC_F_ASYNC_INIT_THREAD), "ASYNC_init_thread"},
     {ERR_FUNC(ASYNC_F_ASYNC_JOB_NEW), "async_job_new"},
     {ERR_FUNC(ASYNC_F_ASYNC_PAUSE_JOB), "ASYNC_pause_job"},
     {ERR_FUNC(ASYNC_F_ASYNC_START_FUNC), "async_start_func"},

crypto/async/async_locl.h

@@ -86,6 +86,7 @@ struct async_pool_st {
     size_t max_size;
 };
 
+int async_thread_local_init(void);
 void async_start_func(void);
 int async_pipe(OSSL_ASYNC_FD *pipefds);
 int async_close_fd(OSSL_ASYNC_FD fd);

doc/crypto/ASYNC_start_job.pod

@@ -2,17 +2,20 @@
 
 =head1 NAME
 
-ASYNC_init_pool, ASYNC_free_pool, ASYNC_start_job, ASYNC_pause_job,
+ASYNC_init, ASYNC_cleanup, ASYNC_init_thread, ASYNC_cleanup_thread,
-ASYNC_in_job, ASYNC_get_wait_fd, ASYNC_get_current_job, ASYNC_wake,
+ASYNC_start_job, ASYNC_pause_job, ASYNC_in_job, ASYNC_get_wait_fd,
-ASYNC_clear_wake, ASYNC_block_pause, ASYNC_unblock_pause - asynchronous job
+ASYNC_get_current_job, ASYNC_wake, ASYNC_clear_wake, ASYNC_block_pause,
-management functions
+ASYNC_unblock_pause - asynchronous job management functions
 
 =head1 SYNOPSIS
 
  #include <openssl/async.h>
 
- int ASYNC_init_pool(size_t max_size, size_t init_size);
+ int ASYNC_init(int init_thread, size_t max_size, size_t init_size);
- void ASYNC_free_pool(void);
+ void ASYNC_cleanup(int cleanupthread);
+
+ int ASYNC_init_thread(size_t max_size, size_t init_size);
+ void ASYNC_cleanup_thread(void);
 
  int ASYNC_start_job(ASYNC_JOB **job, int *ret, int (*func)(void *),
                      void *args, size_t size);
@@ -37,23 +40,28 @@ efficiency reasons, jobs can be created up front and reused many times. They are
 held in a pool until they are needed, at which point they are removed from the
 pool, used, and then returned to the pool when the job completes. Before using
 any of the asynchronous job functions, user code should first call
-ASYNC_init_pool(). If the user application is multi-threaded, then this should
+ASYNC_init(). If the user application is multi-threaded, then
-be done for each thread that will initiate asynchronous jobs. Before user code
+ASYNC_init_thread() should be called for each thread that will initiate
-exits it should free the pool up (for each thread where a pool was initialised)
+asynchronous jobs. If the B<init_thread> parameter to ASYNC_init() is non-zero
-using ASYNC_free_pool(). No asynchronous jobs must be outstanding for the thread
+then ASYNC_init_thread is automatically called for the current thread. Before
-when ASYNC_free_pool() is called. Failing to ensure this will result in memory
+user code exits it should free up resources for each thread that was initialised
-leaks.
+using ASYNC_cleanup_thread(). No asynchronous jobs must be outstanding for the thread
+when ASYNC_cleanup_thread() is called. Failing to ensure this will result in memory
+leaks. Additionally an application should call ASYNC_cleanup() when all
+asynchronous work is complete across all threads. If B<cleanupthread> is
+non-zero then ASYNC_cleanup_thread() is automatically called for the current
+thread.
 
 The B<max_size> argument limits the number of ASYNC_JOBs that will be held in
 the pool. If B<max_size> is set to 0 then no upper limit is set. When an
 ASYNC_JOB is needed but there are none available in the pool already then one
 will be automatically created, as long as the total of ASYNC_JOBs managed by the
 pool does not exceed B<max_size>. When the pool is first initialised
-B<init_size> ASYNC_JOBs will be created immediately. If ASYNC_init_pool() is not
+B<init_size> ASYNC_JOBs will be created immediately. If ASYNC_init_thread() is
-called before the pool is first used then it will be called automatically with a
+not called before the pool is first used then it will be called automatically
-B<max_size> of 0 (no upper limit) and an B<init_size> of 0 (no ASYNC_JOBs
+with a B<max_size> of 0 (no upper limit) and an B<init_size> of 0 (no ASYNC_JOBs
 created up front). If a pool is created in this way it must still be cleaned up
-with an explicit call to ASYNC_free_pool().
+with an explicit call to ASYNC_cleanup_thread().
 
 An asynchronous job is started by calling the ASYNC_start_job() function.
 Initially B<*job> should be NULL. B<ret> should point to a location where the
@@ -141,7 +149,7 @@ occur.
 
 =head1 RETURN VALUES
 
-ASYNC_init_pool returns 1 on success or 0 otherwise.
+ASYNC_init and ASYNC_init_thread return 1 on success or 0 otherwise.
 
 ASYNC_start_job returns one of ASYNC_ERR, ASYNC_NO_JOBS, ASYNC_PAUSE or
 ASYNC_FINISH as described above.
@@ -209,7 +217,7 @@ The following example demonstrates how to use most of the core async APIs:
       * We're only expecting 1 job to be used here so we're only creating
       * a pool of 1
       */
-     if (!ASYNC_init_pool(1, 1)) {
+     if (!ASYNC_init(1, 1, 1)) {
          printf("Error creating pool\n");
          goto end;
      }
@@ -240,7 +248,7 @@ The following example demonstrates how to use most of the core async APIs:
 
  end:
      printf("Finishing\n");
-     ASYNC_free_pool();
+     ASYNC_cleanup(1);
 
      return 0;
  }
@@ -262,8 +270,8 @@ L<crypto(3)>, L<ERR_print_errors(3)>
 
 =head1 HISTORY
 
-ASYNC_init_pool, ASYNC_free_pool, ASYNC_start_job, ASYNC_pause_job,
+ASYNC_init, ASYNC_init_thread, ASYNC_cleanup, ASYNC_cleanup_thread,
-ASYNC_get_wait_fd, ASYNC_get_current_job, ASYNC_wake, ASYNC_clear_wake were
+ASYNC_start_job, ASYNC_pause_job, ASYNC_get_wait_fd, ASYNC_get_current_job,
-first added to OpenSSL 1.1.0.
+ASYNC_wake, ASYNC_clear_wake were first added to OpenSSL 1.1.0.
 
 =cut

include/openssl/async.h

@@ -75,8 +75,10 @@ typedef struct async_job_st ASYNC_JOB;
 #define ASYNC_PAUSE    2
 #define ASYNC_FINISH   3
 
-int ASYNC_init_pool(size_t max_size, size_t init_size);
+int ASYNC_init(int init_thread, size_t max_size, size_t init_size);
-void ASYNC_free_pool(void);
+void ASYNC_cleanup(int cleanupthread);
+int ASYNC_init_thread(size_t max_size, size_t init_size);
+void ASYNC_cleanup_thread(void);
 
 int ASYNC_start_job(ASYNC_JOB **job, int *ret, int (*func)(void *),
                          void *args, size_t size);
@@ -100,7 +102,7 @@ void ERR_load_ASYNC_strings(void);
 
 /* Function codes. */
 # define ASYNC_F_ASYNC_CTX_NEW                            100
-# define ASYNC_F_ASYNC_INIT_POOL                          101
+# define ASYNC_F_ASYNC_INIT_THREAD                        101
 # define ASYNC_F_ASYNC_JOB_NEW                            102
 # define ASYNC_F_ASYNC_PAUSE_JOB                          103
 # define ASYNC_F_ASYNC_START_FUNC                         104

test/asynctest.c

@@ -62,7 +62,7 @@
 #include <openssl/crypto.h>
 #include <../apps/apps.h>
 
-#ifdef OPENSSL_SYS_UNIX
+#if defined(OPENSSL_SYS_UNIX) && defined(OPENSSL_THREADS)
 # include <unistd.h>
 # if _POSIX_VERSION >= 200112L
 #  define ASYNC_POSIX
@@ -124,12 +124,12 @@ static int blockpause(void *args)
     return 1;
 }
 
-static int test_ASYNC_init_pool()
+static int test_ASYNC_init()
 {
     ASYNC_JOB *job1 = NULL, *job2 = NULL, *job3 = NULL;
     int funcret1, funcret2, funcret3;
 
-    if (       !ASYNC_init_pool(2, 0)
+    if (       !ASYNC_init(1, 2, 0)
             || ASYNC_start_job(&job1, &funcret1, only_pause, NULL, 0)
                 != ASYNC_PAUSE
             || ASYNC_start_job(&job2, &funcret2, only_pause, NULL, 0)
@@ -147,12 +147,12 @@ static int test_ASYNC_init_pool()
             || funcret1 != 1
             || funcret2 != 1
             || funcret3 != 1) {
-        fprintf(stderr, "test_ASYNC_init_pool() failed\n");
+        fprintf(stderr, "test_ASYNC_init() failed\n");
-        ASYNC_free_pool();
+        ASYNC_cleanup(1);
         return 0;
     }
 
-    ASYNC_free_pool();
+    ASYNC_cleanup(1);
     return 1;
 }
 
@@ -163,18 +163,18 @@ static int test_ASYNC_start_job()
 
     ctr = 0;
 
-    if (       !ASYNC_init_pool(1, 0)
+    if (       !ASYNC_init(1, 1, 0)
             || ASYNC_start_job(&job, &funcret, add_two, NULL, 0) != ASYNC_PAUSE
             || ctr != 1
             || ASYNC_start_job(&job, &funcret, add_two, NULL, 0) != ASYNC_FINISH
             || ctr != 2
             || funcret != 2) {
         fprintf(stderr, "test_ASYNC_start_job() failed\n");
-        ASYNC_free_pool();
+        ASYNC_cleanup(1);
         return 0;
     }
 
-    ASYNC_free_pool();
+    ASYNC_cleanup(1);
     return 1;
 }
 
@@ -185,7 +185,7 @@ static int test_ASYNC_get_current_job()
 
     currjob = NULL;
 
-    if (       !ASYNC_init_pool(1, 0)
+    if (       !ASYNC_init(1, 1, 0)
             || ASYNC_start_job(&job, &funcret, save_current, NULL, 0)
                 != ASYNC_PAUSE
             || currjob != job
@@ -193,11 +193,11 @@ static int test_ASYNC_get_current_job()
                 != ASYNC_FINISH
             || funcret != 1) {
         fprintf(stderr, "test_ASYNC_get_current_job() failed\n");
-        ASYNC_free_pool();
+        ASYNC_cleanup(1);
         return 0;
     }
 
-    ASYNC_free_pool();
+    ASYNC_cleanup(1);
     return 1;
 }
 
@@ -230,7 +230,7 @@ static int test_ASYNC_get_wait_fd()
     int funcret;
     OSSL_ASYNC_FD fd;
 
-    if (       !ASYNC_init_pool(1, 0)
+    if (       !ASYNC_init(1, 1, 0)
             || ASYNC_start_job(&job, &funcret, wake, NULL, 0)
                 != ASYNC_PAUSE
             || (fd = ASYNC_get_wait_fd(job)) < 0
@@ -246,11 +246,11 @@ static int test_ASYNC_get_wait_fd()
                 != ASYNC_FINISH
             || funcret != 1) {
         fprintf(stderr, "test_ASYNC_get_wait_fd() failed\n");
-        ASYNC_free_pool();
+        ASYNC_cleanup(1);
         return 0;
     }
 
-    ASYNC_free_pool();
+    ASYNC_cleanup(1);
     return 1;
 }
 
@@ -259,18 +259,18 @@ static int test_ASYNC_block_pause()
     ASYNC_JOB *job = NULL;
     int funcret;
 
-    if (       !ASYNC_init_pool(1, 0)
+    if (       !ASYNC_init(1, 1, 0)
             || ASYNC_start_job(&job, &funcret, blockpause, NULL, 0)
                 != ASYNC_PAUSE
             || ASYNC_start_job(&job, &funcret, blockpause, NULL, 0)
                 != ASYNC_FINISH
             || funcret != 1) {
         fprintf(stderr, "test_ASYNC_block_pause() failed\n");
-        ASYNC_free_pool();
+        ASYNC_cleanup(1);
         return 0;
     }
 
-    ASYNC_free_pool();
+    ASYNC_cleanup(1);
     return 1;
 }
 
@@ -286,7 +286,7 @@ int main(int argc, char **argv)
     CRYPTO_set_mem_debug_options(V_CRYPTO_MDEBUG_ALL);
     CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
 
-    if (       !test_ASYNC_init_pool()
+    if (       !test_ASYNC_init()
             || !test_ASYNC_start_job()
             || !test_ASYNC_get_current_job()
             || !test_ASYNC_get_wait_fd()

util/libeay.num

@@ -4652,8 +4652,8 @@ TS_CONF_set_signer_digest               5011	EXIST::FUNCTION:
 ENGINE_load_dasync                      5012	EXIST::FUNCTION:ENGINE,STATIC_ENGINE
 ASYNC_pause_job                         5013	EXIST::FUNCTION:
 ASYNC_start_job                         5014	EXIST::FUNCTION:
-ASYNC_init_pool                         5015	EXIST::FUNCTION:
+ASYNC_init_thread                       5015	EXIST::FUNCTION:
-ASYNC_free_pool                         5016	EXIST::FUNCTION:
+ASYNC_cleanup_thread                    5016	EXIST::FUNCTION:
 ASYNC_wake                              5017	EXIST::FUNCTION:
 ASYNC_clear_wake                        5018	EXIST::FUNCTION:
 ASYNC_get_current_job                   5019	EXIST::FUNCTION: