Implement [util.smartptr.shared.atomic]. This is the last unimplemented

section in libc++.  This requires a recompiled dylib.  Failure to rebuild
the dylib will result in a link-time error if and only if the functions from
[util.smartptr.shared.atomic] are used.

The implementation is not lock free.  After considerable thought, I know of no
way to make the implementation lock free.  Ideas welcome along that front.  But
changing the ABI of shared_ptr is not on the table at this point.

The mutex used to lock these function is encapsulated by std::__sp_mut.  The
only thing the client knows about std::__sp_mut is that it has a void* data
member, can't be constructed, and has lock and unlock members.  Within the
binary __sp_mut is currently implemented as a pointer to a std::mutex.  That can
change in the future without disturbing the ABI (as long as sizeof(__sp_mut)
remains constant.

I specifically did not make __sp_mut a spin lock as I have a pathological
distrust of spin locks.  Testing on OS X reveals that the use of std::mutex in
this role is not a large performance penalty as long as the contention for the
mutex is low (more likely to get the lock than to have to wait).  In the future
we can still make __sp_mut a spin lock if that is what is desired (without ABI
damage).

The dylib contains 16 __sp_mut's to be chosen based on the hash of the address
of the shared_ptr.  The constant 16 is a ball-park reasonable space/time
tradeoff.

std::hash<T*> was changed to call __murmur2_or_cityhash, instead of the identity
function.  I had thought we had already done this, but I was mistaken.

All of this is under #if __has_feature(cxx_atomic) even though the
implementation is not lock free, because the signatures require access to
std::memory_order, which is currently available only under
__has_feature(cxx_atomic).

git-svn-id: https://llvm.org/svn/llvm-project/libcxx/trunk@160940 91177308-0d34-0410-b5e6-96231b3b80d8

include/memory

@@ -602,6 +602,10 @@ void* align(size_t alignment, size_t size, void*& ptr, size_t& space);
     #include <cassert>
 #endif
 
+#if __has_feature(cxx_atomic)
+#  include <atomic>
+#endif
+
 #include <__undef_min_max>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -3387,8 +3391,19 @@ struct __scalar_hash<_Tp, 4>
 
 template<class _Tp>
 struct _LIBCPP_VISIBLE hash<_Tp*>
-    : public __scalar_hash<_Tp*>
+    : public unary_function<_Tp*, size_t>
 {
+    _LIBCPP_INLINE_VISIBILITY
+    size_t operator()(_Tp* __v) const _NOEXCEPT
+    {
+        union
+        {
+            _Tp* __t;
+            size_t __a;
+        } __u;
+        __u.__t = __v;
+        return __murmur2_or_cityhash<size_t>()(&__u, sizeof(__u));
+    }
 };
 
 template <class _Tp, class _Dp>
@@ -3934,6 +3949,10 @@ public:
         _LIBCPP_INLINE_VISIBILITY
         bool owner_before(weak_ptr<_Up> const& __p) const
         {return __cntrl_ < __p.__cntrl_;}
+    _LIBCPP_INLINE_VISIBILITY
+    bool
+    __owner_equivalent(const shared_ptr& __p) const
+        {return __cntrl_ == __p.__cntrl_;}
 
 #ifndef _LIBCPP_NO_RTTI
     template <class _Dp>
@@ -5245,6 +5264,134 @@ inline _LIBCPP_INLINE_VISIBILITY
 basic_ostream<_CharT, _Traits>&
 operator<<(basic_ostream<_CharT, _Traits>& __os, shared_ptr<_Yp> const& __p);
 
+#if __has_feature(cxx_atomic)
+
+class __sp_mut
+{
+    void* _;
+public:
+    void lock() _NOEXCEPT;
+    void unlock() _NOEXCEPT;
+
+private:
+    _LIBCPP_CONSTEXPR __sp_mut(void*) _NOEXCEPT;
+    __sp_mut(const __sp_mut&);
+    __sp_mut& operator=(const __sp_mut&);
+
+    friend __sp_mut& __get_sp_mut(const void*);
+};
+
+__sp_mut& __get_sp_mut(const void*);
+
+template <class _Tp>
+inline _LIBCPP_INLINE_VISIBILITY
+bool
+atomic_is_lock_free(const shared_ptr<_Tp>*)
+{
+    return false;
+}
+
+template <class _Tp>
+shared_ptr<_Tp>
+atomic_load(const shared_ptr<_Tp>* __p)
+{
+    __sp_mut& __m = __get_sp_mut(__p);
+    __m.lock();
+    shared_ptr<_Tp> __q = *__p;
+    __m.unlock();
+    return __q;
+}
+  
+template <class _Tp>
+inline _LIBCPP_INLINE_VISIBILITY
+shared_ptr<_Tp>
+atomic_load_explicit(const shared_ptr<_Tp>* __p, memory_order)
+{
+    return atomic_load(__p);
+}
+
+template <class _Tp>
+void
+atomic_store(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r)
+{
+    __sp_mut& __m = __get_sp_mut(__p);
+    __m.lock();
+    __p->swap(__r);
+    __m.unlock();
+}
+
+template <class _Tp>
+inline _LIBCPP_INLINE_VISIBILITY
+void
+atomic_store_explicit(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r, memory_order)
+{
+    atomic_store(__p, __r);
+}
+
+template <class _Tp>
+shared_ptr<_Tp>
+atomic_exchange(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r)
+{
+    __sp_mut& __m = __get_sp_mut(__p);
+    __m.lock();
+    __p->swap(__r);
+    __m.unlock();
+    return __r;
+}
+  
+template <class _Tp>
+inline _LIBCPP_INLINE_VISIBILITY
+shared_ptr<_Tp>
+atomic_exchange_explicit(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r, memory_order)
+{
+    return atomic_exchange(__p, __r);
+}
+
+template <class _Tp>
+bool
+atomic_compare_exchange_strong(shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v, shared_ptr<_Tp> __w)
+{
+    __sp_mut& __m = __get_sp_mut(__p);
+    __m.lock();
+    if (__p->__owner_equivalent(*__v))
+    {
+        *__p = __w;
+        __m.unlock();
+        return true;
+    }
+    *__v = *__p;
+    __m.unlock();
+    return false;
+}
+
+template <class _Tp>
+inline _LIBCPP_INLINE_VISIBILITY
+bool
+atomic_compare_exchange_weak(shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v, shared_ptr<_Tp> __w)
+{
+    return atomic_compare_exchange_strong(__p, __v, __w);
+}
+
+template <class _Tp>
+inline _LIBCPP_INLINE_VISIBILITY
+bool
+atomic_compare_exchange_strong_explicit(shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v,
+                                        shared_ptr<_Tp> __w, memory_order, memory_order)
+{
+    return atomic_compare_exchange_strong(__p, __v, __w);
+}
+
+template <class _Tp>
+inline _LIBCPP_INLINE_VISIBILITY
+bool
+atomic_compare_exchange_weak_explicit(shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v,
+                                      shared_ptr<_Tp> __w, memory_order, memory_order)
+{
+    return atomic_compare_exchange_weak(__p, __v, __w);
+}
+
+#endif  // __has_feature(cxx_atomic)
+
 //enum class
 struct _LIBCPP_VISIBLE pointer_safety
 {

src/memory.cpp

@@ -9,6 +9,7 @@
 
 #define _LIBCPP_BUILDING_MEMORY
 #include "memory"
+#include "mutex"
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -117,6 +118,40 @@ __shared_weak_count::__get_deleter(const type_info&) const _NOEXCEPT
 
 #endif  // _LIBCPP_NO_RTTI
 
+static const std::size_t __sp_mut_count = 16;
+static mutex mut_back[__sp_mut_count];
+
+_LIBCPP_CONSTEXPR __sp_mut::__sp_mut(void* p) _NOEXCEPT
+   : _(p)
+{
+}
+
+void
+__sp_mut::lock() _NOEXCEPT
+{
+    reinterpret_cast<mutex*>(_)->lock();
+}
+
+void
+__sp_mut::unlock() _NOEXCEPT
+{
+    reinterpret_cast<mutex*>(_)->unlock();
+}
+
+__sp_mut&
+__get_sp_mut(const void* p)
+{
+    static __sp_mut muts[__sp_mut_count] 
+    {
+        &mut_back[ 0], &mut_back[ 1], &mut_back[ 2], &mut_back[ 3],
+        &mut_back[ 4], &mut_back[ 5], &mut_back[ 6], &mut_back[ 7],
+        &mut_back[ 8], &mut_back[ 9], &mut_back[10], &mut_back[11],
+        &mut_back[12], &mut_back[13], &mut_back[14], &mut_back[15]
+    };
+    return muts[hash<const void*>()(p) & (__sp_mut_count-1)];
+}
+
+
 void
 declare_reachable(void*)
 {

test/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong.pass.cpp

@@ -0,0 +1,46 @@
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// <memory>
+
+// shared_ptr
+
+// template <class T>
+// bool
+// atomic_compare_exchange_strong(shared_ptr<T>* p, shared_ptr<T>* v,
+//                                shared_ptr<T> w);
+
+#include <memory>
+#include <cassert>
+
+int main()
+{
+#if __has_feature(cxx_atomic)
+    {
+        std::shared_ptr<int> p(new int(4));
+        std::shared_ptr<int> v(new int(3));
+        std::shared_ptr<int> w(new int(2));
+        bool b = std::atomic_compare_exchange_strong(&p, &v, w);
+        assert(b == false);
+        assert(*p == 4);
+        assert(*v == 4);
+        assert(*w == 2);
+    }
+    {
+        std::shared_ptr<int> p(new int(4));
+        std::shared_ptr<int> v = p;
+        std::shared_ptr<int> w(new int(2));
+        bool b = std::atomic_compare_exchange_strong(&p, &v, w);
+        assert(b == true);
+        assert(*p == 2);
+        assert(*v == 4);
+        assert(*w == 2);
+    }
+#endif
+}

test/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.pass.cpp

@@ -0,0 +1,51 @@
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// <memory>
+
+// shared_ptr
+
+// template <class T>
+// bool
+// atomic_compare_exchange_strong_explicit(shared_ptr<T>* p, shared_ptr<T>* v,
+//                                         shared_ptr<T> w, memory_order success,
+//                                         memory_order failure);
+
+#include <memory>
+#include <cassert>
+
+int main()
+{
+#if __has_feature(cxx_atomic)
+    {
+        std::shared_ptr<int> p(new int(4));
+        std::shared_ptr<int> v(new int(3));
+        std::shared_ptr<int> w(new int(2));
+        bool b = std::atomic_compare_exchange_strong_explicit(&p, &v, w,
+                                                              std::memory_order_seq_cst,
+                                                              std::memory_order_seq_cst);
+        assert(b == false);
+        assert(*p == 4);
+        assert(*v == 4);
+        assert(*w == 2);
+    }
+    {
+        std::shared_ptr<int> p(new int(4));
+        std::shared_ptr<int> v = p;
+        std::shared_ptr<int> w(new int(2));
+        bool b = std::atomic_compare_exchange_strong_explicit(&p, &v, w,
+                                                              std::memory_order_seq_cst,
+                                                              std::memory_order_seq_cst);
+        assert(b == true);
+        assert(*p == 2);
+        assert(*v == 4);
+        assert(*w == 2);
+    }
+#endif
+}

test/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak.pass.cpp

@@ -0,0 +1,46 @@
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// <memory>
+
+// shared_ptr
+
+// template <class T>
+// bool
+// atomic_compare_exchange_weak(shared_ptr<T>* p, shared_ptr<T>* v,
+//                              shared_ptr<T> w);
+
+#include <memory>
+#include <cassert>
+
+int main()
+{
+#if __has_feature(cxx_atomic)
+    {
+        std::shared_ptr<int> p(new int(4));
+        std::shared_ptr<int> v(new int(3));
+        std::shared_ptr<int> w(new int(2));
+        bool b = std::atomic_compare_exchange_weak(&p, &v, w);
+        assert(b == false);
+        assert(*p == 4);
+        assert(*v == 4);
+        assert(*w == 2);
+    }
+    {
+        std::shared_ptr<int> p(new int(4));
+        std::shared_ptr<int> v = p;
+        std::shared_ptr<int> w(new int(2));
+        bool b = std::atomic_compare_exchange_weak(&p, &v, w);
+        assert(b == true);
+        assert(*p == 2);
+        assert(*v == 4);
+        assert(*w == 2);
+    }
+#endif
+}

test/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.pass.cpp

@@ -0,0 +1,51 @@
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// <memory>
+
+// shared_ptr
+
+// template <class T>
+// bool
+// atomic_compare_exchange_weak_explicit(shared_ptr<T>* p, shared_ptr<T>* v,
+//                                       shared_ptr<T> w, memory_order success,
+//                                       memory_order failure);
+
+#include <memory>
+#include <cassert>
+
+int main()
+{
+#if __has_feature(cxx_atomic)
+    {
+        std::shared_ptr<int> p(new int(4));
+        std::shared_ptr<int> v(new int(3));
+        std::shared_ptr<int> w(new int(2));
+        bool b = std::atomic_compare_exchange_weak_explicit(&p, &v, w,
+                                                            std::memory_order_seq_cst,
+                                                            std::memory_order_seq_cst);
+        assert(b == false);
+        assert(*p == 4);
+        assert(*v == 4);
+        assert(*w == 2);
+    }
+    {
+        std::shared_ptr<int> p(new int(4));
+        std::shared_ptr<int> v = p;
+        std::shared_ptr<int> w(new int(2));
+        bool b = std::atomic_compare_exchange_weak_explicit(&p, &v, w,
+                                                            std::memory_order_seq_cst,
+                                                            std::memory_order_seq_cst);
+        assert(b == true);
+        assert(*p == 2);
+        assert(*v == 4);
+        assert(*w == 2);
+    }
+#endif
+}

test/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange.pass.cpp

@@ -0,0 +1,32 @@
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// <memory>
+
+// shared_ptr
+
+// template <class T>
+// shared_ptr<T>
+// atomic_exchange(shared_ptr<T>* p, shared_ptr<T> r)
+
+#include <memory>
+#include <cassert>
+
+int main()
+{
+#if __has_feature(cxx_atomic)
+    {
+        std::shared_ptr<int> p(new int(4));
+        std::shared_ptr<int> r(new int(3));
+        r = std::atomic_exchange(&p, r);
+        assert(*p == 3);
+        assert(*r == 4);
+    }
+#endif
+}

test/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange_explicit.pass.cpp

@@ -0,0 +1,32 @@
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// <memory>
+
+// shared_ptr
+
+// template <class T>
+// shared_ptr<T>
+// atomic_exchange_explicit(shared_ptr<T>* p, shared_ptr<T> r)
+
+#include <memory>
+#include <cassert>
+
+int main()
+{
+#if __has_feature(cxx_atomic)
+    {
+        std::shared_ptr<int> p(new int(4));
+        std::shared_ptr<int> r(new int(3));
+        r = std::atomic_exchange_explicit(&p, r, std::memory_order_seq_cst);
+        assert(*p == 3);
+        assert(*r == 4);
+    }
+#endif
+}

test/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_is_lock_free.pass.cpp

@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// <memory>
+
+// shared_ptr
+
+// template<class T>
+// bool
+// atomic_is_lock_free(const shared_ptr<T>* p);
+
+#include <memory>
+#include <cassert>
+
+int main()
+{
+#if __has_feature(cxx_atomic)
+    {
+        const std::shared_ptr<int> p(new int(3));
+        assert(std::atomic_is_lock_free(&p) == false);
+    }
+#endif
+}

test/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load.pass.cpp

@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// <memory>
+
+// shared_ptr
+
+// template <class T>
+// shared_ptr<T>
+// atomic_load(const shared_ptr<T>* p)
+
+#include <memory>
+#include <cassert>
+
+int main()
+{
+#if __has_feature(cxx_atomic)
+    {
+        std::shared_ptr<int> p(new int(3));
+        std::shared_ptr<int> q = std::atomic_load(&p);
+        assert(*q == *p);
+    }
+#endif
+}

test/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load_explicit.pass.cpp

@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// <memory>
+
+// shared_ptr
+
+// template <class T>
+// shared_ptr<T>
+// atomic_load_explicit(const shared_ptr<T>* p, memory_order mo)
+
+#include <memory>
+#include <cassert>
+
+int main()
+{
+#if __has_feature(cxx_atomic)
+    {
+        const std::shared_ptr<int> p(new int(3));
+        std::shared_ptr<int> q = std::atomic_load_explicit(&p, std::memory_order_relaxed);
+        assert(*q == *p);
+    }
+#endif
+}

test/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store.pass.cpp

@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// <memory>
+
+// shared_ptr
+
+// template <class T>
+// void
+// atomic_store(shared_ptr<T>* p, shared_ptr<T> r)
+
+#include <memory>
+#include <cassert>
+
+int main()
+{
+#if __has_feature(cxx_atomic)
+    {
+        std::shared_ptr<int> p;
+        std::shared_ptr<int> r(new int(3));
+        std::atomic_store(&p, r);
+        assert(*p == *r);
+    }
+#endif
+}

test/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store_explicit.pass.cpp

@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// <memory>
+
+// shared_ptr
+
+// template <class T>
+// void
+// atomic_store_explicit(shared_ptr<T>* p, shared_ptr<T> r, memory_order mo)
+
+#include <memory>
+#include <cassert>
+
+int main()
+{
+#if __has_feature(cxx_atomic)
+    {
+        std::shared_ptr<int> p;
+        std::shared_ptr<int> r(new int(3));
+        std::atomic_store_explicit(&p, r, std::memory_order_seq_cst);
+        assert(*p == *r);
+    }
+#endif
+}