From 81cffd1f2c7140a4394cb427f0cfbfffe735820c Mon Sep 17 00:00:00 2001 From: "hclam@chromium.org" Date: Wed, 5 Dec 2012 22:51:01 +0000 Subject: [PATCH] Port Chromium's atomicops to WebRTC Porting Chromium's base/atomicops.h and friends into WebRTC. Included the original unit test in Chromium. Review URL: https://webrtc-codereview.appspot.com/964026 git-svn-id: http://webrtc.googlecode.com/svn/trunk@3241 4adac7df-926f-26a2-2b94-8c16560cd09d --- webrtc/system_wrappers/interface/atomicops.h | 148 ++++++++++ .../atomicops_internals_atomicword_compat.h | 103 +++++++ .../interface/atomicops_internals_gcc.h | 108 +++++++ .../interface/atomicops_internals_mac.h | 201 +++++++++++++ .../interface/atomicops_internals_mips_gcc.h | 164 +++++++++++ .../interface/atomicops_internals_x86_gcc.h | 264 ++++++++++++++++++ .../interface/atomicops_internals_x86_msvc.h | 184 ++++++++++++ .../source/atomicops_internals_x86_gcc.cc | 107 +++++++ .../source/atomicops_unittest.cc | 250 +++++++++++++++++ .../source/system_wrappers.gyp | 9 + 10 files changed, 1538 insertions(+) create mode 100644 webrtc/system_wrappers/interface/atomicops.h create mode 100644 webrtc/system_wrappers/interface/atomicops_internals_atomicword_compat.h create mode 100644 webrtc/system_wrappers/interface/atomicops_internals_gcc.h create mode 100644 webrtc/system_wrappers/interface/atomicops_internals_mac.h create mode 100644 webrtc/system_wrappers/interface/atomicops_internals_mips_gcc.h create mode 100644 webrtc/system_wrappers/interface/atomicops_internals_x86_gcc.h create mode 100644 webrtc/system_wrappers/interface/atomicops_internals_x86_msvc.h create mode 100644 webrtc/system_wrappers/source/atomicops_internals_x86_gcc.cc create mode 100644 webrtc/system_wrappers/source/atomicops_unittest.cc diff --git a/webrtc/system_wrappers/interface/atomicops.h b/webrtc/system_wrappers/interface/atomicops.h new file mode 100644 index 000000000..22e0fc121 --- /dev/null +++ b/webrtc/system_wrappers/interface/atomicops.h @@ -0,0 +1,148 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file under third_party_mods/chromium or at: +// http://src.chromium.org/svn/trunk/src/LICENSE + +#ifndef WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_H_ +#define WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_H_ + +#if defined(_MSC_VER) +#include /* intptr_t */ +#endif + +#include "typedefs.h" + +namespace webrtc { + +// Extracted from Chromium's src/base/atomicops.h. + +// +// The routines exported by this module are subtle. If you use them, even if +// you get the code right, it will depend on careful reasoning about atomicity +// and memory ordering; it will be less readable, and harder to maintain. If +// you plan to use these routines, you should have a good reason, such as solid +// evidence that performance would otherwise suffer, or there being no +// alternative. You should assume only properties explicitly guaranteed by the +// specifications in this file. You are almost certainly _not_ writing code +// just for the x86; if you assume x86 semantics, x86 hardware bugs and +// implementations on other archtectures will cause your code to break. If you +// do not know what you are doing, avoid these routines, and use a Mutex. +// +// It is incorrect to make direct assignments to/from an atomic variable. +// You should use one of the Load or Store routines. The NoBarrier +// versions are provided when no barriers are needed: +// NoBarrier_Store() +// NoBarrier_Load() +// Although there are currently no compiler enforcement, you are encouraged +// to use these. +// + +namespace subtle { + +typedef int32_t Atomic32; +#ifdef WEBRTC_ARCH_64_BITS +typedef intptr_t Atomic64; +#endif // WEBRTC_ARCH_64_BITS + +// Use AtomicWord for a machine-sized pointer. It will use the Atomic32 or +// Atomic64 routines below, depending on your architecture. +typedef intptr_t AtomicWord; + +// Atomically execute: +// result = *ptr; +// if (*ptr == old_value) +// *ptr = new_value; +// return result; +// +// I.e., replace "*ptr" with "new_value" if "*ptr" used to be "old_value". +// Always return the old value of "*ptr" +// +// This routine implies no memory barriers. +Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value); + +// Atomically store new_value into *ptr, returning the previous value held in +// *ptr. This routine implies no memory barriers. +Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, Atomic32 new_value); + +// Atomically increment *ptr by "increment". Returns the new value of +// *ptr with the increment applied. This routine implies no memory barriers. +Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, Atomic32 increment); + +Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, + Atomic32 increment); + +// These following lower-level operations are typically useful only to people +// implementing higher-level synchronization operations like spinlocks, +// mutexes, and condition-variables. They combine CompareAndSwap(), a load, or +// a store with appropriate memory-ordering instructions. "Acquire" operations +// ensure that no later memory access can be reordered ahead of the operation. +// "Release" operations ensure that no previous memory access can be reordered +// after the operation. "Barrier" operations have both "Acquire" and "Release" +// semantics. A MemoryBarrier() has "Barrier" semantics, but does no memory +// access. +Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value); +Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value); + +void MemoryBarrier(); +void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value); +void Acquire_Store(volatile Atomic32* ptr, Atomic32 value); +void Release_Store(volatile Atomic32* ptr, Atomic32 value); + +Atomic32 NoBarrier_Load(volatile const Atomic32* ptr); +Atomic32 Acquire_Load(volatile const Atomic32* ptr); +Atomic32 Release_Load(volatile const Atomic32* ptr); + +// 64-bit atomic operations (only available on 64-bit processors). +#ifdef WEBRTC_ARCH_64_BITS +Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value); +Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, Atomic64 new_value); +Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, Atomic64 increment); +Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, Atomic64 increment); + +Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value); +Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value); +void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value); +void Acquire_Store(volatile Atomic64* ptr, Atomic64 value); +void Release_Store(volatile Atomic64* ptr, Atomic64 value); +Atomic64 NoBarrier_Load(volatile const Atomic64* ptr); +Atomic64 Acquire_Load(volatile const Atomic64* ptr); +Atomic64 Release_Load(volatile const Atomic64* ptr); +#endif // WEBRTC_ARCH_64_BITS + +} // namespace webrtc::subtle +} // namespace webrtc + +// Include our platform specific implementation. +#if defined(WEBRTC_WIN) && defined(_MSC_VER) && defined(WEBRTC_ARCH_X86_FAMILY) +#include "webrtc/system_wrappers/interface/atomicops_internals_x86_msvc.h" +#elif defined(WEBRTC_MAC) +#include "webrtc/system_wrappers/interface/atomicops_internals_mac.h" +#elif defined(__GNUC__) && defined(__ARMEL__) +#include "webrtc/system_wrappers/interface/atomicops_internals_gcc.h" +#elif defined(__GNUC__) && defined(WEBRTC_ARCH_X86_FAMILY) +#include "webrtc/system_wrappers/interface/atomicops_internals_x86_gcc.h" +#elif defined(__GNUC__) && defined(__MIPSEL__) +#include "webrtc/system_wrappers/interface/atomicops_internals_mips_gcc.h" +#else +#error "Atomic operations are not supported on your platform" +#endif + +// On some platforms we need additional declarations to make +// AtomicWord compatible with our other Atomic* types. +#if defined(WEBRTC_MAC) || defined(__OpenBSD__) +#include "webrtc/system_wrappers/interface/atomicops_internals_atomicword_compat.h" +#endif + +#endif // WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_H_ diff --git a/webrtc/system_wrappers/interface/atomicops_internals_atomicword_compat.h b/webrtc/system_wrappers/interface/atomicops_internals_atomicword_compat.h new file mode 100644 index 000000000..28ffe8cf8 --- /dev/null +++ b/webrtc/system_wrappers/interface/atomicops_internals_atomicword_compat.h @@ -0,0 +1,103 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file under third_party_mods/chromium or at: +// http://src.chromium.org/svn/trunk/src/LICENSE + +#ifndef WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_INTERNALS_ATOMICWORD_COMPAT_H_ +#define WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_INTERNALS_ATOMICWORD_COMPAT_H_ + +// Extracted from Chromium's src/base/atomicops_internals_atomicword_compat.h. + +// This file is an internal atomic implementation, use atomicops.h instead. + +// AtomicWord is a synonym for intptr_t, and Atomic32 is a synonym for int32, +// which in turn means int. On some LP32 platforms, intptr_t is an int, but +// on others, it's a long. When AtomicWord and Atomic32 are based on different +// fundamental types, their pointers are incompatible. +// +// This file defines function overloads to allow both AtomicWord and Atomic32 +// data to be used with this interface. +// +// On LP64 platforms, AtomicWord and Atomic64 are both always long, +// so this problem doesn't occur. + +#if !defined(WEBRTC_ARCH_64_BITS) + +namespace webrtc { +namespace subtle { + +inline AtomicWord NoBarrier_CompareAndSwap(volatile AtomicWord* ptr, + AtomicWord old_value, + AtomicWord new_value) { + return NoBarrier_CompareAndSwap( + reinterpret_cast(ptr), old_value, new_value); +} + +inline AtomicWord NoBarrier_AtomicExchange(volatile AtomicWord* ptr, + AtomicWord new_value) { + return NoBarrier_AtomicExchange( + reinterpret_cast(ptr), new_value); +} + +inline AtomicWord NoBarrier_AtomicIncrement(volatile AtomicWord* ptr, + AtomicWord increment) { + return NoBarrier_AtomicIncrement( + reinterpret_cast(ptr), increment); +} + +inline AtomicWord Barrier_AtomicIncrement(volatile AtomicWord* ptr, + AtomicWord increment) { + return Barrier_AtomicIncrement( + reinterpret_cast(ptr), increment); +} + +inline AtomicWord Acquire_CompareAndSwap(volatile AtomicWord* ptr, + AtomicWord old_value, + AtomicWord new_value) { + return webrtc::subtle::Acquire_CompareAndSwap( + reinterpret_cast(ptr), old_value, new_value); +} + +inline AtomicWord Release_CompareAndSwap(volatile AtomicWord* ptr, + AtomicWord old_value, + AtomicWord new_value) { + return webrtc::subtle::Release_CompareAndSwap( + reinterpret_cast(ptr), old_value, new_value); +} + +inline void NoBarrier_Store(volatile AtomicWord *ptr, AtomicWord value) { + NoBarrier_Store( + reinterpret_cast(ptr), value); +} + +inline void Acquire_Store(volatile AtomicWord* ptr, AtomicWord value) { + return webrtc::subtle::Acquire_Store( + reinterpret_cast(ptr), value); +} + +inline void Release_Store(volatile AtomicWord* ptr, AtomicWord value) { + return webrtc::subtle::Release_Store( + reinterpret_cast(ptr), value); +} + +inline AtomicWord NoBarrier_Load(volatile const AtomicWord *ptr) { + return NoBarrier_Load( + reinterpret_cast(ptr)); +} + +inline AtomicWord Acquire_Load(volatile const AtomicWord* ptr) { + return webrtc::subtle::Acquire_Load( + reinterpret_cast(ptr)); +} + +inline AtomicWord Release_Load(volatile const AtomicWord* ptr) { + return webrtc::subtle::Release_Load( + reinterpret_cast(ptr)); +} + +} // namespace webrtc::subtle +} // namespace webrtc + +#endif // !defined(WEBRTC_ARCH_64_BITS) + +#endif // WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_INTERNALS_ATOMICWORD_COMPAT_H_ diff --git a/webrtc/system_wrappers/interface/atomicops_internals_gcc.h b/webrtc/system_wrappers/interface/atomicops_internals_gcc.h new file mode 100644 index 000000000..01294f680 --- /dev/null +++ b/webrtc/system_wrappers/interface/atomicops_internals_gcc.h @@ -0,0 +1,108 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file under third_party_mods/chromium or at: +// http://src.chromium.org/svn/trunk/src/LICENSE + +#ifndef WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_INTERNALS_GCC_H_ +#define WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_INTERNALS_GCC_H_ + +// Extracted from Chromium's src/base/atomicops_internals_gcc.h. + +// This file is an internal atomic implementation, include atomicops.h +// instead. This file is for platforms that use GCC intrinsics rather than +// platform-specific assembly code for atomic operations. + +namespace webrtc { +namespace subtle { + +inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + Atomic32 prev_value; + do { + if (__sync_bool_compare_and_swap(ptr, old_value, new_value)) + return old_value; + prev_value = *ptr; + } while (prev_value == old_value); + return prev_value; +} + +inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, + Atomic32 new_value) { + Atomic32 old_value; + do { + old_value = *ptr; + } while (!__sync_bool_compare_and_swap(ptr, old_value, new_value)); + return old_value; +} + +inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, + Atomic32 increment) { + return Barrier_AtomicIncrement(ptr, increment); +} + +inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, + Atomic32 increment) { + for (;;) { + // Atomic exchange the old value with an incremented one. + Atomic32 old_value = *ptr; + Atomic32 new_value = old_value + increment; + if (__sync_bool_compare_and_swap(ptr, old_value, new_value)) { + // The exchange took place as expected. + return new_value; + } + // Otherwise, *ptr changed mid-loop and we need to retry. + } +} + +inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + // Since NoBarrier_CompareAndSwap uses __sync_bool_compare_and_swap, which + // is a full memory barrier, none is needed here or below in Release. + return NoBarrier_CompareAndSwap(ptr, old_value, new_value); +} + +inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + return NoBarrier_CompareAndSwap(ptr, old_value, new_value); +} + +inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { + *ptr = value; +} + +inline void MemoryBarrier() { + __sync_synchronize(); +} + +inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { + *ptr = value; + MemoryBarrier(); +} + +inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { + MemoryBarrier(); + *ptr = value; +} + +inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { + return *ptr; +} + +inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { + Atomic32 value = *ptr; + MemoryBarrier(); + return value; +} + +inline Atomic32 Release_Load(volatile const Atomic32* ptr) { + MemoryBarrier(); + return *ptr; +} + +} // namespace webrtc::subtle +} // namespace webrtc + +#endif // WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_INTERNALS_GCC_H_ diff --git a/webrtc/system_wrappers/interface/atomicops_internals_mac.h b/webrtc/system_wrappers/interface/atomicops_internals_mac.h new file mode 100644 index 000000000..1d11d5d27 --- /dev/null +++ b/webrtc/system_wrappers/interface/atomicops_internals_mac.h @@ -0,0 +1,201 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file under third_party_mods/chromium or at: +// http://src.chromium.org/svn/trunk/src/LICENSE + +#ifndef WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_INTERNALS_MAC_H_ +#define WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_INTERNALS_MAC_H_ + +// Extracted from Chromium's src/base/atomicops_internals_mac.h. + +// This file is an internal atomic implementation, use +// webrtc/system_wrappers/interface/atomicops.h instead. + +#include + +namespace webrtc { +namespace subtle { + +inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32 *ptr, + Atomic32 old_value, + Atomic32 new_value) { + Atomic32 prev_value; + do { + if (OSAtomicCompareAndSwap32(old_value, new_value, + const_cast(ptr))) { + return old_value; + } + prev_value = *ptr; + } while (prev_value == old_value); + return prev_value; +} + +inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32 *ptr, + Atomic32 new_value) { + Atomic32 old_value; + do { + old_value = *ptr; + } while (!OSAtomicCompareAndSwap32(old_value, new_value, + const_cast(ptr))); + return old_value; +} + +inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32 *ptr, + Atomic32 increment) { + return OSAtomicAdd32(increment, const_cast(ptr)); +} + +inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32 *ptr, + Atomic32 increment) { + return OSAtomicAdd32Barrier(increment, const_cast(ptr)); +} + +inline void MemoryBarrier() { + OSMemoryBarrier(); +} + +inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32 *ptr, + Atomic32 old_value, + Atomic32 new_value) { + Atomic32 prev_value; + do { + if (OSAtomicCompareAndSwap32Barrier(old_value, new_value, + const_cast(ptr))) { + return old_value; + } + prev_value = *ptr; + } while (prev_value == old_value); + return prev_value; +} + +inline Atomic32 Release_CompareAndSwap(volatile Atomic32 *ptr, + Atomic32 old_value, + Atomic32 new_value) { + return Acquire_CompareAndSwap(ptr, old_value, new_value); +} + +inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { + *ptr = value; +} + +inline void Acquire_Store(volatile Atomic32 *ptr, Atomic32 value) { + *ptr = value; + MemoryBarrier(); +} + +inline void Release_Store(volatile Atomic32 *ptr, Atomic32 value) { + MemoryBarrier(); + *ptr = value; +} + +inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { + return *ptr; +} + +inline Atomic32 Acquire_Load(volatile const Atomic32 *ptr) { + Atomic32 value = *ptr; + MemoryBarrier(); + return value; +} + +inline Atomic32 Release_Load(volatile const Atomic32 *ptr) { + MemoryBarrier(); + return *ptr; +} + +#ifdef __LP64__ + +// 64-bit implementation on 64-bit platform + +inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64 *ptr, + Atomic64 old_value, + Atomic64 new_value) { + Atomic64 prev_value; + do { + if (OSAtomicCompareAndSwap64(old_value, new_value, + reinterpret_cast(ptr))) { + return old_value; + } + prev_value = *ptr; + } while (prev_value == old_value); + return prev_value; +} + +inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64 *ptr, + Atomic64 new_value) { + Atomic64 old_value; + do { + old_value = *ptr; + } while (!OSAtomicCompareAndSwap64(old_value, new_value, + reinterpret_cast(ptr))); + return old_value; +} + +inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64 *ptr, + Atomic64 increment) { + return OSAtomicAdd64(increment, reinterpret_cast(ptr)); +} + +inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64 *ptr, + Atomic64 increment) { + return OSAtomicAdd64Barrier(increment, + reinterpret_cast(ptr)); +} + +inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64 *ptr, + Atomic64 old_value, + Atomic64 new_value) { + Atomic64 prev_value; + do { + if (OSAtomicCompareAndSwap64Barrier( + old_value, new_value, reinterpret_cast(ptr))) { + return old_value; + } + prev_value = *ptr; + } while (prev_value == old_value); + return prev_value; +} + +inline Atomic64 Release_CompareAndSwap(volatile Atomic64 *ptr, + Atomic64 old_value, + Atomic64 new_value) { + // The lib kern interface does not distinguish between + // Acquire and Release memory barriers; they are equivalent. + return Acquire_CompareAndSwap(ptr, old_value, new_value); +} + +inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { + *ptr = value; +} + +inline void Acquire_Store(volatile Atomic64 *ptr, Atomic64 value) { + *ptr = value; + MemoryBarrier(); +} + +inline void Release_Store(volatile Atomic64 *ptr, Atomic64 value) { + MemoryBarrier(); + *ptr = value; +} + +inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { + return *ptr; +} + +inline Atomic64 Acquire_Load(volatile const Atomic64 *ptr) { + Atomic64 value = *ptr; + MemoryBarrier(); + return value; +} + +inline Atomic64 Release_Load(volatile const Atomic64 *ptr) { + MemoryBarrier(); + return *ptr; +} + +#endif // defined(__LP64__) + +} // namespace webrtc::subtle +} // namespace webrtc + +#endif // WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_INTERNALS_MAC_H_ diff --git a/webrtc/system_wrappers/interface/atomicops_internals_mips_gcc.h b/webrtc/system_wrappers/interface/atomicops_internals_mips_gcc.h new file mode 100644 index 000000000..52a096ab7 --- /dev/null +++ b/webrtc/system_wrappers/interface/atomicops_internals_mips_gcc.h @@ -0,0 +1,164 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file under third_party_mods/chromium or at: +// http://src.chromium.org/svn/trunk/src/LICENSE + +#ifndef WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_INTERNALS_MIPS_GCC_H_ +#define WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_INTERNALS_MIPS_GCC_H_ + +// Extracted from Chromium's src/base/atomicops_internals_mips_gcc.h. + +// This file is an internal atomic implementation, use atomicops.h instead. +// +// LinuxKernelCmpxchg and Barrier_AtomicIncrement are from Google Gears. + +#define ATOMICOPS_COMPILER_BARRIER() __asm__ __volatile__("" : : : "memory") + +namespace webrtc { +namespace subtle { + +// Atomically execute: +// result = *ptr; +// if (*ptr == old_value) +// *ptr = new_value; +// return result; +// +// I.e., replace "*ptr" with "new_value" if "*ptr" used to be "old_value". +// Always return the old value of "*ptr" +// +// This routine implies no memory barriers. +inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + Atomic32 prev, tmp; + __asm__ __volatile__(".set push\n" + ".set noreorder\n" + "1:\n" + "ll %0, %5\n" // prev = *ptr + "bne %0, %3, 2f\n" // if (prev != old_value) goto 2 + "move %2, %4\n" // tmp = new_value + "sc %2, %1\n" // *ptr = tmp (with atomic check) + "beqz %2, 1b\n" // start again on atomic error + "nop\n" // delay slot nop + "2:\n" + ".set pop\n" + : "=&r" (prev), "=m" (*ptr), "=&r" (tmp) + : "Ir" (old_value), "r" (new_value), "m" (*ptr) + : "memory"); + return prev; +} + +// Atomically store new_value into *ptr, returning the previous value held in +// *ptr. This routine implies no memory barriers. +inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, + Atomic32 new_value) { + Atomic32 temp, old; + __asm__ __volatile__(".set push\n" + ".set noreorder\n" + "1:\n" + "ll %1, %2\n" // old = *ptr + "move %0, %3\n" // temp = new_value + "sc %0, %2\n" // *ptr = temp (with atomic check) + "beqz %0, 1b\n" // start again on atomic error + "nop\n" // delay slot nop + ".set pop\n" + : "=&r" (temp), "=&r" (old), "=m" (*ptr) + : "r" (new_value), "m" (*ptr) + : "memory"); + + return old; +} + +// Atomically increment *ptr by "increment". Returns the new value of +// *ptr with the increment applied. This routine implies no memory barriers. +inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, + Atomic32 increment) { + Atomic32 temp, temp2; + + __asm__ __volatile__(".set push\n" + ".set noreorder\n" + "1:\n" + "ll %0, %2\n" // temp = *ptr + "addu %1, %0, %3\n" // temp2 = temp + increment + "sc %1, %2\n" // *ptr = temp2 (with atomic check) + "beqz %1, 1b\n" // start again on atomic error + "addu %1, %0, %3\n" // temp2 = temp + increment + ".set pop\n" + : "=&r" (temp), "=&r" (temp2), "=m" (*ptr) + : "Ir" (increment), "m" (*ptr) + : "memory"); + // temp2 now holds the final value. + return temp2; +} + +inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, + Atomic32 increment) { + ATOMICOPS_COMPILER_BARRIER(); + Atomic32 res = NoBarrier_AtomicIncrement(ptr, increment); + ATOMICOPS_COMPILER_BARRIER(); + return res; +} + +// "Acquire" operations +// ensure that no later memory access can be reordered ahead of the operation. +// "Release" operations ensure that no previous memory access can be reordered +// after the operation. "Barrier" operations have both "Acquire" and "Release" +// semantics. A MemoryBarrier() has "Barrier" semantics, but does no memory +// access. +inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + ATOMICOPS_COMPILER_BARRIER(); + Atomic32 res = NoBarrier_CompareAndSwap(ptr, old_value, new_value); + ATOMICOPS_COMPILER_BARRIER(); + return res; +} + +inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + ATOMICOPS_COMPILER_BARRIER(); + Atomic32 res = NoBarrier_CompareAndSwap(ptr, old_value, new_value); + ATOMICOPS_COMPILER_BARRIER(); + return res; +} + +inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { + *ptr = value; +} + +inline void MemoryBarrier() { + __asm__ __volatile__("sync" : : : "memory"); +} + +inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { + *ptr = value; + MemoryBarrier(); +} + +inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { + MemoryBarrier(); + *ptr = value; +} + +inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { + return *ptr; +} + +inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { + Atomic32 value = *ptr; + MemoryBarrier(); + return value; +} + +inline Atomic32 Release_Load(volatile const Atomic32* ptr) { + MemoryBarrier(); + return *ptr; +} + +} // namespace webrtc::subtle +} // namespace webrtc + +#undef ATOMICOPS_COMPILER_BARRIER + +#endif // WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_INTERNALS_MIPS_GCC_H_ diff --git a/webrtc/system_wrappers/interface/atomicops_internals_x86_gcc.h b/webrtc/system_wrappers/interface/atomicops_internals_x86_gcc.h new file mode 100644 index 000000000..625e71593 --- /dev/null +++ b/webrtc/system_wrappers/interface/atomicops_internals_x86_gcc.h @@ -0,0 +1,264 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file under third_party_mods/chromium or at: +// http://src.chromium.org/svn/trunk/src/LICENSE + +#ifndef WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_INTERNALS_X86_GCC_H_ +#define WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_INTERNALS_X86_GCC_H_ + +// Extracted from Chromium's src/base/atomicops_internals_x86_gcc.h. + +// This file is an internal atomic implementation, use atomicops.h instead. + +struct AtomicOps_x86CPUFeatureStruct { + bool has_amd_lock_mb_bug; // Processor has AMD memory-barrier bug; do lfence + // after acquire compare-and-swap. + bool has_sse2; // Processor has SSE2. +}; +extern struct AtomicOps_x86CPUFeatureStruct AtomicOps_Internalx86CPUFeatures; + +#define ATOMICOPS_COMPILER_BARRIER() __asm__ __volatile__("" : : : "memory") + +namespace webrtc { +namespace subtle { + +// 32-bit low-level operations on any platform. + +inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + Atomic32 prev; + __asm__ __volatile__("lock; cmpxchgl %1,%2" + : "=a" (prev) + : "q" (new_value), "m" (*ptr), "0" (old_value) + : "memory"); + return prev; +} + +inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, + Atomic32 new_value) { + __asm__ __volatile__("xchgl %1,%0" // The lock prefix is implicit for xchg. + : "=r" (new_value) + : "m" (*ptr), "0" (new_value) + : "memory"); + return new_value; // Now it's the previous value. +} + +inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, + Atomic32 increment) { + Atomic32 temp = increment; + __asm__ __volatile__("lock; xaddl %0,%1" + : "+r" (temp), "+m" (*ptr) + : : "memory"); + // temp now holds the old value of *ptr + return temp + increment; +} + +inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, + Atomic32 increment) { + Atomic32 temp = increment; + __asm__ __volatile__("lock; xaddl %0,%1" + : "+r" (temp), "+m" (*ptr) + : : "memory"); + // temp now holds the old value of *ptr + if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { + __asm__ __volatile__("lfence" : : : "memory"); + } + return temp + increment; +} + +inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + Atomic32 x = NoBarrier_CompareAndSwap(ptr, old_value, new_value); + if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { + __asm__ __volatile__("lfence" : : : "memory"); + } + return x; +} + +inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + return NoBarrier_CompareAndSwap(ptr, old_value, new_value); +} + +inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { + *ptr = value; +} + +#if defined(__x86_64__) + +// 64-bit implementations of memory barrier can be simpler, because it +// "mfence" is guaranteed to exist. +inline void MemoryBarrier() { + __asm__ __volatile__("mfence" : : : "memory"); +} + +inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { + *ptr = value; + MemoryBarrier(); +} + +#else + +inline void MemoryBarrier() { + if (AtomicOps_Internalx86CPUFeatures.has_sse2) { + __asm__ __volatile__("mfence" : : : "memory"); + } else { // mfence is faster but not present on PIII + Atomic32 x = 0; + NoBarrier_AtomicExchange(&x, 0); // acts as a barrier on PIII + } +} + +inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { + if (AtomicOps_Internalx86CPUFeatures.has_sse2) { + *ptr = value; + __asm__ __volatile__("mfence" : : : "memory"); + } else { + NoBarrier_AtomicExchange(ptr, value); + // acts as a barrier on PIII + } +} +#endif + +inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { + ATOMICOPS_COMPILER_BARRIER(); + *ptr = value; // An x86 store acts as a release barrier. + // See comments in Atomic64 version of Release_Store(), below. +} + +inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { + return *ptr; +} + +inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { + Atomic32 value = *ptr; // An x86 load acts as a acquire barrier. + // See comments in Atomic64 version of Release_Store(), below. + ATOMICOPS_COMPILER_BARRIER(); + return value; +} + +inline Atomic32 Release_Load(volatile const Atomic32* ptr) { + MemoryBarrier(); + return *ptr; +} + +#if defined(__x86_64__) + +// 64-bit low-level operations on 64-bit platform. + +inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + Atomic64 prev; + __asm__ __volatile__("lock; cmpxchgq %1,%2" + : "=a" (prev) + : "q" (new_value), "m" (*ptr), "0" (old_value) + : "memory"); + return prev; +} + +inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, + Atomic64 new_value) { + __asm__ __volatile__("xchgq %1,%0" // The lock prefix is implicit for xchg. + : "=r" (new_value) + : "m" (*ptr), "0" (new_value) + : "memory"); + return new_value; // Now it's the previous value. +} + +inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, + Atomic64 increment) { + Atomic64 temp = increment; + __asm__ __volatile__("lock; xaddq %0,%1" + : "+r" (temp), "+m" (*ptr) + : : "memory"); + // temp now contains the previous value of *ptr + return temp + increment; +} + +inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, + Atomic64 increment) { + Atomic64 temp = increment; + __asm__ __volatile__("lock; xaddq %0,%1" + : "+r" (temp), "+m" (*ptr) + : : "memory"); + // temp now contains the previous value of *ptr + if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { + __asm__ __volatile__("lfence" : : : "memory"); + } + return temp + increment; +} + +inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { + *ptr = value; +} + +inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { + *ptr = value; + MemoryBarrier(); +} + +inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { + ATOMICOPS_COMPILER_BARRIER(); + + *ptr = value; // An x86 store acts as a release barrier + // for current AMD/Intel chips as of Jan 2008. + // See also Acquire_Load(), below. + + // When new chips come out, check: + // IA-32 Intel Architecture Software Developer's Manual, Volume 3: + // System Programming Guide, Chatper 7: Multiple-processor management, + // Section 7.2, Memory Ordering. + // Last seen at: + // http://developer.intel.com/design/pentium4/manuals/index_new.htm + // + // x86 stores/loads fail to act as barriers for a few instructions (clflush + // maskmovdqu maskmovq movntdq movnti movntpd movntps movntq) but these are + // not generated by the compiler, and are rare. Users of these instructions + // need to know about cache behaviour in any case since all of these involve + // either flushing cache lines or non-temporal cache hints. +} + +inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { + return *ptr; +} + +inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { + Atomic64 value = *ptr; // An x86 load acts as a acquire barrier, + // for current AMD/Intel chips as of Jan 2008. + // See also Release_Store(), above. + ATOMICOPS_COMPILER_BARRIER(); + return value; +} + +inline Atomic64 Release_Load(volatile const Atomic64* ptr) { + MemoryBarrier(); + return *ptr; +} + +inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + Atomic64 x = NoBarrier_CompareAndSwap(ptr, old_value, new_value); + if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { + __asm__ __volatile__("lfence" : : : "memory"); + } + return x; +} + +inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + return NoBarrier_CompareAndSwap(ptr, old_value, new_value); +} + +#endif // defined(__x86_64__) + +} // namespace webrtc::subtle +} // namespace webrtc + +#undef ATOMICOPS_COMPILER_BARRIER + +#endif // WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_INTERNALS_X86_GCC_H_ diff --git a/webrtc/system_wrappers/interface/atomicops_internals_x86_msvc.h b/webrtc/system_wrappers/interface/atomicops_internals_x86_msvc.h new file mode 100644 index 000000000..dcb4e70a3 --- /dev/null +++ b/webrtc/system_wrappers/interface/atomicops_internals_x86_msvc.h @@ -0,0 +1,184 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file under third_party_mods/chromium or at: +// http://src.chromium.org/svn/trunk/src/LICENSE + +#ifndef WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_INTERNALS_X86_MSVC_H_ +#define WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_INTERNALS_X86_MSVC_H_ + +// Extracted from Chromium's src/base/atomicops_internals_x86_msvc.h. + +// This file is an internal atomic implementation, use atomicops.h instead. + +#include +#include "compile_assert.h" + +namespace webrtc { +namespace subtle { + +inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + LONG result = InterlockedCompareExchange( + reinterpret_cast(ptr), + static_cast(new_value), + static_cast(old_value)); + return static_cast(result); +} + +inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, + Atomic32 new_value) { + LONG result = InterlockedExchange( + reinterpret_cast(ptr), + static_cast(new_value)); + return static_cast(result); +} + +inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, + Atomic32 increment) { + return InterlockedExchangeAdd( + reinterpret_cast(ptr), + static_cast(increment)) + increment; +} + +inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, + Atomic32 increment) { + return Barrier_AtomicIncrement(ptr, increment); +} + +#if !(defined(_MSC_VER) && _MSC_VER >= 1400) +#error "We require at least vs2005 for MemoryBarrier" +#endif +inline void MemoryBarrier() { + // We use MemoryBarrier from WinNT.h + ::MemoryBarrier(); +} + +inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + return NoBarrier_CompareAndSwap(ptr, old_value, new_value); +} + +inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + return NoBarrier_CompareAndSwap(ptr, old_value, new_value); +} + +inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { + *ptr = value; +} + +inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { + NoBarrier_AtomicExchange(ptr, value); + // acts as a barrier in this implementation +} + +inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { + *ptr = value; // works w/o barrier for current Intel chips as of June 2005 + // See comments in Atomic64 version of Release_Store() below. +} + +inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { + return *ptr; +} + +inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { + Atomic32 value = *ptr; + return value; +} + +inline Atomic32 Release_Load(volatile const Atomic32* ptr) { + MemoryBarrier(); + return *ptr; +} + +#if defined(_WIN64) + +// 64-bit low-level operations on 64-bit platform. + +COMPILE_ASSERT(sizeof(Atomic64) == sizeof(PVOID), atomic_word_is_atomic); + +inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + PVOID result = InterlockedCompareExchangePointer( + reinterpret_cast(ptr), + reinterpret_cast(new_value), reinterpret_cast(old_value)); + return reinterpret_cast(result); +} + +inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, + Atomic64 new_value) { + PVOID result = InterlockedExchangePointer( + reinterpret_cast(ptr), + reinterpret_cast(new_value)); + return reinterpret_cast(result); +} + +inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, + Atomic64 increment) { + return InterlockedExchangeAdd64( + reinterpret_cast(ptr), + static_cast(increment)) + increment; +} + +inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, + Atomic64 increment) { + return Barrier_AtomicIncrement(ptr, increment); +} + +inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { + *ptr = value; +} + +inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { + NoBarrier_AtomicExchange(ptr, value); + // acts as a barrier in this implementation +} + +inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { + *ptr = value; // works w/o barrier for current Intel chips as of June 2005 + + // When new chips come out, check: + // IA-32 Intel Architecture Software Developer's Manual, Volume 3: + // System Programming Guide, Chatper 7: Multiple-processor management, + // Section 7.2, Memory Ordering. + // Last seen at: + // http://developer.intel.com/design/pentium4/manuals/index_new.htm +} + +inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { + return *ptr; +} + +inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { + Atomic64 value = *ptr; + return value; +} + +inline Atomic64 Release_Load(volatile const Atomic64* ptr) { + MemoryBarrier(); + return *ptr; +} + +inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + return NoBarrier_CompareAndSwap(ptr, old_value, new_value); +} + +inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + return NoBarrier_CompareAndSwap(ptr, old_value, new_value); +} + + +#endif // defined(_WIN64) + +} // namespace webrtc::subtle +} // namespace webrtc + +#endif // WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_INTERNALS_X86_MSVC_H_ diff --git a/webrtc/system_wrappers/source/atomicops_internals_x86_gcc.cc b/webrtc/system_wrappers/source/atomicops_internals_x86_gcc.cc new file mode 100644 index 000000000..4d3452dc7 --- /dev/null +++ b/webrtc/system_wrappers/source/atomicops_internals_x86_gcc.cc @@ -0,0 +1,107 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file under third_party_mods/chromium or at: +// http://src.chromium.org/svn/trunk/src/LICENSE + +// Extracted from Chromium's src/base/atomicops_internals_x86_gcc.cc. + +// This module gets enough CPU information to optimize the +// atomicops module on x86. + +#include + +#include "webrtc/system_wrappers/interface/atomicops.h" +#include "webrtc/typedefs.h" + +// This file only makes sense with atomicops_internals_x86_gcc.h -- it +// depends on structs that are defined in that file. If atomicops.h +// doesn't sub-include that file, then we aren't needed, and shouldn't +// try to do anything. +#ifdef WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_INTERNALS_X86_GCC_H_ + +// Inline cpuid instruction. In PIC compilations, %ebx contains the address +// of the global offset table. To avoid breaking such executables, this code +// must preserve that register's value across cpuid instructions. +#if defined(__i386__) +#define cpuid(a, b, c, d, inp) \ + asm ("mov %%ebx, %%edi\n" \ + "cpuid\n" \ + "xchg %%edi, %%ebx\n" \ + : "=a" (a), "=D" (b), "=c" (c), "=d" (d) : "a" (inp)) +#elif defined (__x86_64__) +#define cpuid(a, b, c, d, inp) \ + asm ("mov %%rbx, %%rdi\n" \ + "cpuid\n" \ + "xchg %%rdi, %%rbx\n" \ + : "=a" (a), "=D" (b), "=c" (c), "=d" (d) : "a" (inp)) +#endif + +#if defined(cpuid) // initialize the struct only on x86 + +// Set the flags so that code will run correctly and conservatively, so even +// if we haven't been initialized yet, we're probably single threaded, and our +// default values should hopefully be pretty safe. +struct AtomicOps_x86CPUFeatureStruct AtomicOps_Internalx86CPUFeatures = { + false, // bug can't exist before process spawns multiple threads + false, // no SSE2 +}; + +// Initialize the AtomicOps_Internalx86CPUFeatures struct. +static void AtomicOps_Internalx86CPUFeaturesInit() { + uint32_t eax; + uint32_t ebx; + uint32_t ecx; + uint32_t edx; + + // Get vendor string (issue CPUID with eax = 0) + cpuid(eax, ebx, ecx, edx, 0); + char vendor[13]; + memcpy(vendor, &ebx, 4); + memcpy(vendor + 4, &edx, 4); + memcpy(vendor + 8, &ecx, 4); + vendor[12] = 0; + + // get feature flags in ecx/edx, and family/model in eax + cpuid(eax, ebx, ecx, edx, 1); + + int family = (eax >> 8) & 0xf; // family and model fields + int model = (eax >> 4) & 0xf; + if (family == 0xf) { // use extended family and model fields + family += (eax >> 20) & 0xff; + model += ((eax >> 16) & 0xf) << 4; + } + + // Opteron Rev E has a bug in which on very rare occasions a locked + // instruction doesn't act as a read-acquire barrier if followed by a + // non-locked read-modify-write instruction. Rev F has this bug in + // pre-release versions, but not in versions released to customers, + // so we test only for Rev E, which is family 15, model 32..63 inclusive. + if (strcmp(vendor, "AuthenticAMD") == 0 && // AMD + family == 15 && + 32 <= model && model <= 63) { + AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug = true; + } else { + AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug = false; + } + + // edx bit 26 is SSE2 which we use to tell use whether we can use mfence + AtomicOps_Internalx86CPUFeatures.has_sse2 = ((edx >> 26) & 1); +} + +namespace { + +class AtomicOpsx86Initializer { + public: + AtomicOpsx86Initializer() { + AtomicOps_Internalx86CPUFeaturesInit(); + } +}; + +// A global to get use initialized on startup via static initialization :/ +AtomicOpsx86Initializer g_initer; + +} // namespace + +#endif // if x86 + +#endif // if WEBRTC_SYSTEM_WRAPPERS_INTERFACE_ATOMICOPS_INTERNALS_X86_GCC_H_ diff --git a/webrtc/system_wrappers/source/atomicops_unittest.cc b/webrtc/system_wrappers/source/atomicops_unittest.cc new file mode 100644 index 000000000..2b056f3c4 --- /dev/null +++ b/webrtc/system_wrappers/source/atomicops_unittest.cc @@ -0,0 +1,250 @@ +// Copyright (c) 2011 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file under third_party_mods/chromium or at: +// http://src.chromium.org/svn/trunk/src/LICENSE + +#include "webrtc/system_wrappers/interface/atomicops.h" + +#include + +#include "testing/gtest/include/gtest/gtest.h" + +// Extracted from Chromium's src/base/port.h. +// It is included here since it's a small file and only used by this here. +#if defined(_MSC_VER) +#define GG_LONGLONG(x) x##I64 +#define GG_ULONGLONG(x) x##UI64 +#else +#define GG_LONGLONG(x) x##LL +#define GG_ULONGLONG(x) x##ULL +#endif +#define NUM_BITS(T) (sizeof(T) * 8) + +// Extracted from Chromium's src/base/atomicops_unittest.h. + +template +static void TestAtomicIncrement() { + // For now, we just test single threaded execution + + // use a guard value to make sure the NoBarrier_AtomicIncrement doesn't go + // outside the expected address bounds. This is in particular to + // test that some future change to the asm code doesn't cause the + // 32-bit NoBarrier_AtomicIncrement doesn't do the wrong thing on 64-bit + // machines. + struct { + AtomicType prev_word; + AtomicType count; + AtomicType next_word; + } s; + + AtomicType prev_word_value, next_word_value; + memset(&prev_word_value, 0xFF, sizeof(AtomicType)); + memset(&next_word_value, 0xEE, sizeof(AtomicType)); + + s.prev_word = prev_word_value; + s.count = 0; + s.next_word = next_word_value; + + EXPECT_EQ(webrtc::subtle::NoBarrier_AtomicIncrement(&s.count, 1), 1); + EXPECT_EQ(s.count, 1); + EXPECT_EQ(s.prev_word, prev_word_value); + EXPECT_EQ(s.next_word, next_word_value); + + EXPECT_EQ(webrtc::subtle::NoBarrier_AtomicIncrement(&s.count, 2), 3); + EXPECT_EQ(s.count, 3); + EXPECT_EQ(s.prev_word, prev_word_value); + EXPECT_EQ(s.next_word, next_word_value); + + EXPECT_EQ(webrtc::subtle::NoBarrier_AtomicIncrement(&s.count, 3), 6); + EXPECT_EQ(s.count, 6); + EXPECT_EQ(s.prev_word, prev_word_value); + EXPECT_EQ(s.next_word, next_word_value); + + EXPECT_EQ(webrtc::subtle::NoBarrier_AtomicIncrement(&s.count, -3), 3); + EXPECT_EQ(s.count, 3); + EXPECT_EQ(s.prev_word, prev_word_value); + EXPECT_EQ(s.next_word, next_word_value); + + EXPECT_EQ(webrtc::subtle::NoBarrier_AtomicIncrement(&s.count, -2), 1); + EXPECT_EQ(s.count, 1); + EXPECT_EQ(s.prev_word, prev_word_value); + EXPECT_EQ(s.next_word, next_word_value); + + EXPECT_EQ(webrtc::subtle::NoBarrier_AtomicIncrement(&s.count, -1), 0); + EXPECT_EQ(s.count, 0); + EXPECT_EQ(s.prev_word, prev_word_value); + EXPECT_EQ(s.next_word, next_word_value); + + EXPECT_EQ(webrtc::subtle::NoBarrier_AtomicIncrement(&s.count, -1), -1); + EXPECT_EQ(s.count, -1); + EXPECT_EQ(s.prev_word, prev_word_value); + EXPECT_EQ(s.next_word, next_word_value); + + EXPECT_EQ(webrtc::subtle::NoBarrier_AtomicIncrement(&s.count, -4), -5); + EXPECT_EQ(s.count, -5); + EXPECT_EQ(s.prev_word, prev_word_value); + EXPECT_EQ(s.next_word, next_word_value); + + EXPECT_EQ(webrtc::subtle::NoBarrier_AtomicIncrement(&s.count, 5), 0); + EXPECT_EQ(s.count, 0); + EXPECT_EQ(s.prev_word, prev_word_value); + EXPECT_EQ(s.next_word, next_word_value); +} + +template +static void TestCompareAndSwap() { + AtomicType value = 0; + AtomicType prev = webrtc::subtle::NoBarrier_CompareAndSwap(&value, 0, 1); + EXPECT_EQ(1, value); + EXPECT_EQ(0, prev); + + // Use test value that has non-zero bits in both halves, more for testing + // 64-bit implementation on 32-bit platforms. + const AtomicType k_test_val = (GG_ULONGLONG(1) << + (NUM_BITS(AtomicType) - 2)) + 11; + value = k_test_val; + prev = webrtc::subtle::NoBarrier_CompareAndSwap(&value, 0, 5); + EXPECT_EQ(k_test_val, value); + EXPECT_EQ(k_test_val, prev); + + value = k_test_val; + prev = webrtc::subtle::NoBarrier_CompareAndSwap(&value, k_test_val, 5); + EXPECT_EQ(5, value); + EXPECT_EQ(k_test_val, prev); +} + + +template +static void TestAtomicExchange() { + AtomicType value = 0; + AtomicType new_value = webrtc::subtle::NoBarrier_AtomicExchange(&value, 1); + EXPECT_EQ(1, value); + EXPECT_EQ(0, new_value); + + // Use test value that has non-zero bits in both halves, more for testing + // 64-bit implementation on 32-bit platforms. + const AtomicType k_test_val = (GG_ULONGLONG(1) << + (NUM_BITS(AtomicType) - 2)) + 11; + value = k_test_val; + new_value = webrtc::subtle::NoBarrier_AtomicExchange(&value, k_test_val); + EXPECT_EQ(k_test_val, value); + EXPECT_EQ(k_test_val, new_value); + + value = k_test_val; + new_value = webrtc::subtle::NoBarrier_AtomicExchange(&value, 5); + EXPECT_EQ(5, value); + EXPECT_EQ(k_test_val, new_value); +} + + +template +static void TestAtomicIncrementBounds() { + // Test at rollover boundary between int_max and int_min + AtomicType test_val = (GG_ULONGLONG(1) << + (NUM_BITS(AtomicType) - 1)); + AtomicType value = -1 ^ test_val; + AtomicType new_value = webrtc::subtle::NoBarrier_AtomicIncrement(&value, 1); + EXPECT_EQ(test_val, value); + EXPECT_EQ(value, new_value); + + webrtc::subtle::NoBarrier_AtomicIncrement(&value, -1); + EXPECT_EQ(-1 ^ test_val, value); + + // Test at 32-bit boundary for 64-bit atomic type. + test_val = GG_ULONGLONG(1) << (NUM_BITS(AtomicType) / 2); + value = test_val - 1; + new_value = webrtc::subtle::NoBarrier_AtomicIncrement(&value, 1); + EXPECT_EQ(test_val, value); + EXPECT_EQ(value, new_value); + + webrtc::subtle::NoBarrier_AtomicIncrement(&value, -1); + EXPECT_EQ(test_val - 1, value); +} + +// Return an AtomicType with the value 0xa5a5a5.. +template +static AtomicType TestFillValue() { + AtomicType val = 0; + memset(&val, 0xa5, sizeof(AtomicType)); + return val; +} + +// This is a simple sanity check that values are correct. Not testing +// atomicity +template +static void TestStore() { + const AtomicType kVal1 = TestFillValue(); + const AtomicType kVal2 = static_cast(-1); + + AtomicType value; + + webrtc::subtle::NoBarrier_Store(&value, kVal1); + EXPECT_EQ(kVal1, value); + webrtc::subtle::NoBarrier_Store(&value, kVal2); + EXPECT_EQ(kVal2, value); + + webrtc::subtle::Acquire_Store(&value, kVal1); + EXPECT_EQ(kVal1, value); + webrtc::subtle::Acquire_Store(&value, kVal2); + EXPECT_EQ(kVal2, value); + + webrtc::subtle::Release_Store(&value, kVal1); + EXPECT_EQ(kVal1, value); + webrtc::subtle::Release_Store(&value, kVal2); + EXPECT_EQ(kVal2, value); +} + +// This is a simple sanity check that values are correct. Not testing +// atomicity +template +static void TestLoad() { + const AtomicType kVal1 = TestFillValue(); + const AtomicType kVal2 = static_cast(-1); + + AtomicType value; + + value = kVal1; + EXPECT_EQ(kVal1, webrtc::subtle::NoBarrier_Load(&value)); + value = kVal2; + EXPECT_EQ(kVal2, webrtc::subtle::NoBarrier_Load(&value)); + + value = kVal1; + EXPECT_EQ(kVal1, webrtc::subtle::Acquire_Load(&value)); + value = kVal2; + EXPECT_EQ(kVal2, webrtc::subtle::Acquire_Load(&value)); + + value = kVal1; + EXPECT_EQ(kVal1, webrtc::subtle::Release_Load(&value)); + value = kVal2; + EXPECT_EQ(kVal2, webrtc::subtle::Release_Load(&value)); +} + +TEST(AtomicOpsTest, Inc) { + TestAtomicIncrement(); + TestAtomicIncrement(); +} + +TEST(AtomicOpsTest, CompareAndSwap) { + TestCompareAndSwap(); + TestCompareAndSwap(); +} + +TEST(AtomicOpsTest, Exchange) { + TestAtomicExchange(); + TestAtomicExchange(); +} + +TEST(AtomicOpsTest, IncrementBounds) { + TestAtomicIncrementBounds(); + TestAtomicIncrementBounds(); +} + +TEST(AtomicOpsTest, Store) { + TestStore(); + TestStore(); +} + +TEST(AtomicOpsTest, Load) { + TestLoad(); + TestLoad(); +} diff --git a/webrtc/system_wrappers/source/system_wrappers.gyp b/webrtc/system_wrappers/source/system_wrappers.gyp index 4ce148bad..67bf7b5ce 100644 --- a/webrtc/system_wrappers/source/system_wrappers.gyp +++ b/webrtc/system_wrappers/source/system_wrappers.gyp @@ -24,6 +24,13 @@ 'sources': [ '../interface/aligned_malloc.h', '../interface/atomic32.h', + '../interface/atomicops.h', + '../interface/atomicops_internals_atomicword_compat.h', + '../interface/atomicops_internals_gcc.h', + '../interface/atomicops_internals_mac.h', + '../interface/atomicops_internals_mips_gcc.h', + '../interface/atomicops_internals_x86_gcc.h', + '../interface/atomicops_internals_x86_msvc.h', '../interface/compile_assert.h', '../interface/condition_variable_wrapper.h', '../interface/cpu_info.h', @@ -53,6 +60,7 @@ 'atomic32_mac.cc', 'atomic32_posix.cc', 'atomic32_win.cc', + 'atomicops_internals_x86_gcc.cc', 'condition_variable.cc', 'condition_variable_posix.cc', 'condition_variable_posix.h', @@ -226,6 +234,7 @@ ], 'sources': [ 'aligned_malloc_unittest.cc', + 'atomicops_unittest.cc', 'condition_variable_unittest.cc', 'cpu_wrapper_unittest.cc', 'cpu_measurement_harness.h',