From 4a6b01f1438507bd3b701a60fb016b5696ee9a47 Mon Sep 17 00:00:00 2001 From: Pavel Andrianov Date: Tue, 10 Jan 2023 11:35:45 +0300 Subject: [PATCH 01/17] Initial implementation of rwlock Change-Id: Ic3d7efd49c99c5fde6cc7ddfee70ed54cfd37c7a --- libpandabase/CMakeLists.txt | 6 +- libpandabase/tests/genmc/condvar_test_1.cpp | 1 + libpandabase/tests/genmc/condvar_test_2.cpp | 1 + libpandabase/tests/genmc/condvar_test_3.cpp | 1 + libpandabase/tests/genmc/rwlock_test_1.cpp | 70 +++++ platforms/unix/libpandabase/futex/fmutex.cpp | 300 ++++++++++++++++++- platforms/unix/libpandabase/futex/fmutex.h | 34 ++- platforms/unix/libpandabase/futex/mutex.cpp | 194 +----------- platforms/unix/libpandabase/futex/mutex.h | 105 +------ 9 files changed, 418 insertions(+), 294 deletions(-) create mode 100644 libpandabase/tests/genmc/rwlock_test_1.cpp diff --git a/libpandabase/CMakeLists.txt b/libpandabase/CMakeLists.txt index d2995a5dc..5e7f71192 100644 --- a/libpandabase/CMakeLists.txt +++ b/libpandabase/CMakeLists.txt @@ -415,10 +415,10 @@ if (DEFINED ENV{GENMC_PATH}) ${PANDA_ROOT}/libpandabase/tests/genmc/mutex_test_2.cpp ${PANDA_ROOT}/libpandabase/tests/genmc/mutex_test_3.cpp ${PANDA_ROOT}/libpandabase/tests/genmc/mutex_test_4.cpp - ${PANDA_ROOT}/libpandabase/tests/genmc/condvar_test_1.cpp - # Takes too much time #8866 + # ${PANDA_ROOT}/libpandabase/tests/genmc/condvar_test_1.cpp # ${PANDA_ROOT}/libpandabase/tests/genmc/condvar_test_2.cpp - ${PANDA_ROOT}/libpandabase/tests/genmc/condvar_test_3.cpp + # ${PANDA_ROOT}/libpandabase/tests/genmc/condvar_test_3.cpp + # ${PANDA_ROOT}/libpandabase/tests/genmc/rwlock_test_1.cpp ) foreach(genmc_test ${GENMC_TESTS}) diff --git a/libpandabase/tests/genmc/condvar_test_1.cpp b/libpandabase/tests/genmc/condvar_test_1.cpp index 578af55e9..cf6454e14 100644 --- a/libpandabase/tests/genmc/condvar_test_1.cpp +++ b/libpandabase/tests/genmc/condvar_test_1.cpp @@ -14,6 +14,7 @@ */ #include "common.h" +#include "genmc.h" // The test checks the work of Wait-SignalOne // Thread1 sets g_shared to 1, then waits, diff --git a/libpandabase/tests/genmc/condvar_test_2.cpp b/libpandabase/tests/genmc/condvar_test_2.cpp index 73f7b159d..5bdbe0450 100644 --- a/libpandabase/tests/genmc/condvar_test_2.cpp +++ b/libpandabase/tests/genmc/condvar_test_2.cpp @@ -14,6 +14,7 @@ */ #include "common.h" +#include "genmc.h" // The test checks the work of Wait-SignalAll // There are two waiters (Thread1) and one waker (Thread2) diff --git a/libpandabase/tests/genmc/condvar_test_3.cpp b/libpandabase/tests/genmc/condvar_test_3.cpp index e245da709..0ca732099 100644 --- a/libpandabase/tests/genmc/condvar_test_3.cpp +++ b/libpandabase/tests/genmc/condvar_test_3.cpp @@ -14,6 +14,7 @@ */ #include "common.h" +#include "genmc.h" // The test checks the work of TimedWait-SignalOne // Thread1 sets g_shared to 1, then waits with timeout, diff --git a/libpandabase/tests/genmc/rwlock_test_1.cpp b/libpandabase/tests/genmc/rwlock_test_1.cpp new file mode 100644 index 000000000..64bc9aa44 --- /dev/null +++ b/libpandabase/tests/genmc/rwlock_test_1.cpp @@ -0,0 +1,70 @@ +/** + * Copyright (c) 2021-2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common.h" + +// The test checks basic work of RWLock +// Reader reads the global variable, then checks its value +// Writer increments the global variable and checks its value + +static struct rw_lock lock; + +static void *Reader(void *arg) +{ + int index = static_cast(reinterpret_cast(arg)); + + RWLockReadLock(&lock); + int tmp = g_shared; + ASSERT(tmp == g_shared); + RWLockUnlock(&lock); + + ASSERT(tmp == 0 || tmp == 1 || tmp == 2); + return nullptr; +} + +static void *Writer(void *arg) +{ + RWLockWriteLock(&lock); + // increment global var and check its value + CheckGlobalVar(g_shared + 1); + RWLockUnlock(&lock); + return nullptr; +} + +int main() +{ + RWLockInit(&lock); + g_shared = 0; + pthread_t t1; + pthread_t t2; + pthread_t t3; + pthread_t t4; + + pthread_create(&t1, nullptr, Reader, nullptr); + pthread_create(&t2, nullptr, Reader, nullptr); + pthread_create(&t3, nullptr, Writer, nullptr); + pthread_create(&t4, nullptr, Writer, nullptr); + + pthread_join(t1, nullptr); + pthread_join(t2, nullptr); + pthread_join(t3, nullptr); + pthread_join(t4, nullptr); + + // Check that writer was finished + ASSERT(g_shared == 2); + + RWLockDestroy(&lock); + return 0; +} \ No newline at end of file diff --git a/platforms/unix/libpandabase/futex/fmutex.cpp b/platforms/unix/libpandabase/futex/fmutex.cpp index a5dbac319..7de8c6c34 100644 --- a/platforms/unix/libpandabase/futex/fmutex.cpp +++ b/platforms/unix/libpandabase/futex/fmutex.cpp @@ -38,6 +38,7 @@ static ATOMIC(bool) DEADLOCK_FLAG = false; #ifdef MC_ON // GenMC does not support syscalls(futex) static ATOMIC_INT FUTEX_SIGNAL; +static ATOMIC_INT SIGNAL; static inline void FutexWait(ATOMIC_INT *m, int v) { @@ -57,6 +58,25 @@ static inline void FutexWake(void) // Atomic with release order reason: mutex synchronization ATOMIC_FETCH_ADD(&FUTEX_SIGNAL, 1, memory_order_release); } + +static inline void SignalWait(ATOMIC_INT *m, int v) +{ + // Atomic with acquire order reason: mutex synchronization + int s = ATOMIC_LOAD(&SIGNAL, memory_order_acquire); + // Atomic with acquire order reason: mutex synchronization + if (ATOMIC_LOAD(m, memory_order_acquire) != v) { + return; + } + // Atomic with acquire order reason: mutex synchronization + while (ATOMIC_LOAD(&SIGNAL, memory_order_acquire) == s) { + } +} + +static inline void SignalWake(void) +{ + // Atomic with release order reason: mutex synchronization + ATOMIC_FETCH_ADD(&SIGNAL, 1, memory_order_release); +} #else // futex() is defined in header, as it is still included in different places #endif @@ -139,6 +159,30 @@ static inline bool WaitBrieflyFor(ATOMIC_INT *addr) return false; } +// GenMC does not support lambdas, have to copy the wait function +static inline bool WaitBrieflyFor(ATOMIC_INT *addr, bool readlock) +{ +#ifndef MC_ON + // We probably don't want to do syscall (switch context) when we use WaitBrieflyFor + static constexpr uint32_t MAX_BACK_OFF = 10; + static constexpr uint32_t maxIter = 50; + for (uint32_t i = 1; i <= maxIter; i++) { + BackOff(MIN(i, MAX_BACK_OFF)); +#endif + // Atomic with relaxed order reason: mutex synchronization + int state = ATOMIC_LOAD(addr, memory_order_relaxed); + if (readlock && state != WRITE_LOCKED) { + return true; + } + if (state == UNLOCKED) { + return true; + } +#ifndef MC_ON + } +#endif + return false; +} + void MutexInit(struct fmutex *const m) { // Atomic with relaxed order reason: mutex synchronization @@ -404,7 +448,7 @@ void Wait(struct CondVar *const cond, struct fmutex *const m) MutexUnlock(m); #ifdef MC_ON - FutexWait(&cond->cond, cur_cond); + SignalWait(&cond->cond, cur_cond); #else // NOLINTNEXTLINE(hicpp-signed-bitwise), NOLINTNEXTLINE(C_RULE_ID_FUNCTION_NESTING_LEVEL) if (futex(GetCondAddr(cond), FUTEX_WAIT_PRIVATE, cur_cond, nullptr, nullptr, 0) != 0) { @@ -451,7 +495,7 @@ bool TimedWait(struct CondVar *const cond, struct fmutex *const m, uint64_t ms, MutexUnlock(m); #ifdef MC_ON - FutexWait(&cond->cond, cur_cond); + SignalWait(&cond->cond, cur_cond); #else int wait_flag; int match_flag; @@ -503,7 +547,7 @@ void SignalCount(struct CondVar *const cond, int32_t to_wake) ATOMIC_FETCH_ADD(&cond->cond, 1, memory_order_relaxed); #ifdef MC_ON - FutexWake(); + SignalWake(); #else if (IsHeld(mutex, current_tid)) { // This thread is owner of current mutex, do requeue to mutex waitqueue @@ -521,6 +565,256 @@ void SignalCount(struct CondVar *const cond, int32_t to_wake) #endif } +// Empty string +int *GetStateAddr(struct rw_lock *const rwlock) +{ + return reinterpret_cast(&rwlock->state); +} + +bool HasExclusiveHolder(struct rw_lock *const rwlock) +{ + // Atomic with relaxed order reason: mutex synchronization + return ATOMIC_LOAD(&rwlock->state, memory_order_relaxed) == WRITE_LOCKED; +} + +void IncrementWaiters(struct rw_lock *const rwlock) +{ + // Atomic with relaxed order reason: mutex synchronization + ATOMIC_FETCH_ADD(&rwlock->waiters, 1, memory_order_relaxed); +} +void DecrementWaiters(struct rw_lock *const rwlock) +{ + // Atomic with relaxed order reason: mutex synchronization + ATOMIC_FETCH_SUB(&rwlock->waiters, 1, memory_order_relaxed); +} + +void RWLockInit(struct rw_lock *const rwlock) +{ + ATOMIC_STORE(&rwlock->state, 0, memory_order_relaxed); + ATOMIC_STORE(&rwlock->waiters, 0, memory_order_relaxed); +} + +void RWLockDestroy(struct rw_lock *const rwlock) +{ +#ifndef PANDA_TARGET_MOBILE + if (!MutexDoNotCheckOnTerminationLoop()) { +#endif // PANDA_TARGET_MOBILE + // Atomic with relaxed order reason: mutex synchronization + if (ATOMIC_LOAD(&rwlock->state, memory_order_relaxed) != 0) { + FAIL_WITH_MESSAGE("RWLock destruction failed; state is non zero!"); + // Atomic with relaxed order reason: mutex synchronization + } else if (ATOMIC_LOAD(&rwlock->waiters, memory_order_relaxed) != 0) { + FAIL_WITH_MESSAGE("RWLock destruction failed; RWLock has waiters!"); + } +#ifndef PANDA_TARGET_MOBILE + } else { + LOG_MESSAGE(WARNING, "Termination loop detected, ignoring RWLock"); + } +#endif // PANDA_TARGET_MOBILE +} + +void HandleReadLockWait(struct rw_lock *const rwlock, int32_t cur_state) +{ + // Wait until RWLock WriteLock is unlocked + if (!WaitBrieflyFor(&rwlock->state, true)) { + // WaitBrieflyFor failed, go to futex wait + IncrementWaiters(rwlock); + // Retry wait until WriteLock not held. + while (cur_state == WRITE_LOCKED) { + // NOLINTNEXTLINE(hicpp-signed-bitwise) +#ifdef MC_ON + FutexWait(&rwlock->state, cur_state); +#else + if (futex(GetStateAddr(rwlock), FUTEX_WAIT_PRIVATE, cur_state, nullptr, nullptr, 0) != 0) { + if ((errno != EAGAIN) && (errno != EINTR)) { + LOG(FATAL, COMMON) << "Futex wait failed!"; + } + } +#endif + // Atomic with relaxed order reason: mutex synchronization + cur_state = ATOMIC_LOAD(&rwlock->state, memory_order_relaxed); + } + DecrementWaiters(rwlock); + } +} + +void RWLockReadLock(struct rw_lock *const rwlock) +{ + bool done = false; + do { + // Atomic with relaxed order reason: mutex synchronization + auto cur_state = ATOMIC_LOAD(&rwlock->state, memory_order_relaxed); + if (LIKELY(cur_state != WRITE_LOCKED)) { + auto new_state = cur_state + READ_INCREMENT; + done = ATOMIC_CAS_WEAK_1(&rwlock->state, cur_state, new_state, memory_order_acquire); + } else { + HandleReadLockWait(rwlock, cur_state); + } + } while (!done); + ASSERT(!HasExclusiveHolder(rwlock)); +} + +void RWLockWriteLock(struct rw_lock *const rwlock) +{ + if (current_tid == 0) { + current_tid = GET_CURRENT_THREAD; + } + bool done = false; + while (!done) { + // Atomic with relaxed order reason: mutex synchronization + auto cur_state = ATOMIC_LOAD(&rwlock->state, memory_order_relaxed); + if (LIKELY(cur_state == UNLOCKED)) { + // Unlocked, can acquire writelock + // Do CAS in case other thread beats us and acquires readlock first + done = ATOMIC_CAS_WEAK_1(&rwlock->state, cur_state, WRITE_LOCKED, memory_order_acquire); + } else { + // Wait until RWLock is unlocked + if (!WaitBrieflyFor(&rwlock->state, false)) { + // WaitBrieflyFor failed, go to futex wait + // Increment waiters count. + IncrementWaiters(rwlock); + // Retry wait until lock not held. If we have more than one reader, cur_state check fail + // doesn't mean this lock is unlocked. + while (cur_state != UNLOCKED) { +#ifdef MC_ON + FutexWait(&rwlock->state, cur_state); +#else + // NOLINTNEXTLINE(hicpp-signed-bitwise) + if (futex(GetStateAddr(rwlock), FUTEX_WAIT_PRIVATE, cur_state, nullptr, nullptr, 0) != 0) { + if ((errno != EAGAIN) && (errno != EINTR)) { + FAIL_WITH_MESSAGE("Futex wait failed!"); + } + } +#endif + // Atomic with relaxed order reason: mutex synchronization + cur_state = ATOMIC_LOAD(&rwlock->state, memory_order_relaxed); + } + DecrementWaiters(rwlock); + } + } + } + // RWLock is held now + // Atomic with relaxed order reason: mutex synchronization + ASSERT(ATOMIC_LOAD(&rwlock->state, memory_order_relaxed) == WRITE_LOCKED); +} + +bool RWLockTryReadLock(struct rw_lock *const rwlock) +{ + bool done = false; + // Atomic with relaxed order reason: mutex synchronization + auto cur_state = ATOMIC_LOAD(&rwlock->state, memory_order_relaxed); + while (!done) { + if (cur_state >= UNLOCKED) { + auto new_state = cur_state + READ_INCREMENT; + // cur_state should be updated with fetched value on fail + done = ATOMIC_CAS_WEAK_1(&rwlock->state, cur_state, new_state, memory_order_acquire); + } else { + // RWLock is Write held, trylock failed. + return false; + } + } + ASSERT(!HasExclusiveHolder(rwlock)); + return true; +} + +bool RWLockTryWriteLock(struct rw_lock *const rwlock) +{ + if (current_tid == 0) { + current_tid = GET_CURRENT_THREAD; + } + bool done = false; + // Atomic with relaxed order reason: mutex synchronization + auto cur_state = ATOMIC_LOAD(&rwlock->state, memory_order_relaxed); + while (!done) { + if (LIKELY(cur_state == UNLOCKED)) { + // Unlocked, can acquire writelock + // Do CAS in case other thread beats us and acquires readlock first + // cur_state should be updated with fetched value on fail + done = ATOMIC_CAS_WEAK_1(&rwlock->state, cur_state, WRITE_LOCKED, memory_order_acquire); + } else { + // RWLock is held, trylock failed. + return false; + } + } + // RWLock is held now + // Atomic with relaxed order reason: mutex synchronization + ASSERT(ATOMIC_LOAD(&rwlock->state, memory_order_relaxed) == WRITE_LOCKED); + return true; +} + +void RWLockReadUnlock(struct rw_lock *const rwlock) +{ + ASSERT(!HasExclusiveHolder(rwlock)); + bool done = false; + // Atomic with relaxed order reason: mutex synchronization + auto cur_state = ATOMIC_LOAD(&rwlock->state, memory_order_relaxed); + do { + if (LIKELY(cur_state > 0)) { + // Reduce state by 1 and do release store. + // waiters load should not be reordered before state, so it's done with seq cst. + auto new_state = cur_state - READ_INCREMENT; + // cur_state should be updated with fetched value on fail + // Atomic with seq_cst order reason: mutex synchronization + done = ATOMIC_CAS_WEAK_1(&rwlock->state, cur_state, new_state, memory_order_release); + if (done && new_state == UNLOCKED) { + // Atomic with seq_cst order reason: mutex synchronization + if (ATOMIC_LOAD(&rwlock->waiters, memory_order_relaxed) > 0) { + // Wake one exclusive waiter as there are now no readers. +#ifdef MC_ON + FutexWake(); +#else + // NOLINTNEXTLINE(hicpp-signed-bitwise) + futex(GetStateAddr(rwlock), FUTEX_WAKE_PRIVATE, WAKE_ALL, nullptr, nullptr, 0); +#endif + } + } + } else { + FAIL_WITH_MESSAGE("RWLock ReadUnlock got unexpected state, RWLock is unlocked?"); + } + } while (!done); +} + +void RWLockWriteUnlock(struct rw_lock *const rwlock) +{ + if (current_tid == 0) { + current_tid = GET_CURRENT_THREAD; + } + ASSERT(HasExclusiveHolder(rwlock)); + + bool done = false; + // Atomic with relaxed order reason: mutex synchronization + int32_t cur_state = ATOMIC_LOAD(&rwlock->state, memory_order_relaxed); + // CAS is weak and might fail, do in loop + while (!done) { + if (LIKELY(cur_state == WRITE_LOCKED)) { + // Change state to unlocked and do release store. + // waiters load should not be reordered before state, so it's done with seq cst. + // cur_state should be updated with fetched value on fail + done = ATOMIC_CAS_WEAK_1(&rwlock->state, cur_state, UNLOCKED, memory_order_release); + } else { + FAIL_WITH_MESSAGE("RWLock WriteUnlock got unexpected state, RWLock is not writelocked?"); + } + } + // We are doing write unlock, all waiters could be ReadLocks so we need to wake all. + // Atomic with seq_cst order reason: mutex synchronization + if (ATOMIC_LOAD(&rwlock->waiters, memory_order_relaxed) > 0) { +#ifdef MC_ON + FutexWake(); +#else + // NOLINTNEXTLINE(hicpp-signed-bitwise) + futex(GetStateAddr(rwlock), FUTEX_WAKE_PRIVATE, WAKE_ALL, nullptr, nullptr, 0); +#endif + } +} + +void RWLockUnlock(struct rw_lock *const rwlock) +{ + if (HasExclusiveHolder(rwlock)) { + RWLockWriteUnlock(rwlock); + } else { + RWLockReadUnlock(rwlock); + } +} #ifndef MC_ON } // namespace panda::os::unix::memory::futex #endif diff --git a/platforms/unix/libpandabase/futex/fmutex.h b/platforms/unix/libpandabase/futex/fmutex.h index 36d08c098..25f64102b 100644 --- a/platforms/unix/libpandabase/futex/fmutex.h +++ b/platforms/unix/libpandabase/futex/fmutex.h @@ -31,13 +31,15 @@ #define ATOMIC_FETCH_SUB(addr, val, mem) atomic_fetch_sub_explicit(addr, val, mem) #define ATOMIC_CAS_WEAK(addr, old_val, new_val, mem1, mem2) \ atomic_compare_exchange_weak_explicit(addr, &old_val, new_val, mem1, mem2) +#define ATOMIC_CAS_WEAK_1(addr, old_val, new_val, mem) \ + atomic_compare_exchange_weak_explicit(addr, &old_val, new_val, mem, mem) #define ASSERT(a) assert(a) #define LIKELY(a) a #define UNLIKELY(a) a -#define MIN(a, b) (((a) < (b)) ? (a) : (b)) -#define MAX(a, b) (((a) > (b)) ? (a) : (b)) +#define ALWAYS_INLINE #else #include +#include #include #include #include @@ -56,12 +58,15 @@ namespace panda::os::unix::memory::futex { // NOLINTNEXTLINE(cppcoreguidelines-macro-usage) #define ATOMIC_CAS_WEAK(addr, old_val, new_val, mem1, mem2) \ (addr)->compare_exchange_weak(old_val, new_val, std::mem1, std::mem2) +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define ATOMIC_CAS_WEAK_1(addr, old_val, new_val, mem) (addr)->compare_exchange_weak(old_val, new_val, std::mem) #endif // Copy of mutex storage, after complete implementation should totally replace mutex::current_tid // NOLINTNEXTLINE(readability-identifier-naming) extern thread_local THREAD_ID current_tid; +// Empty string void MutexInit(struct fmutex *m); void MutexDestroy(struct fmutex *m); bool MutexLock(struct fmutex *m, bool trylock); @@ -126,6 +131,31 @@ __attribute__((visibility("default"))) void Wait(struct CondVar *cond, struct fm __attribute__((visibility("default"))) bool TimedWait(struct CondVar *cond, struct fmutex *m, uint64_t ms, uint64_t ns, bool is_absolute); +static constexpr int32_t WRITE_LOCKED = -1; +static constexpr int32_t UNLOCKED = 0; +static constexpr int32_t READ_INCREMENT = 1; +// Extra padding to make RWLock 16 bytes long +static constexpr size_t PADDING_SIZE = 1; + +struct rw_lock { + // -1 - write locked; 0 - unlocked; > 0 - read locked by state_ owners. + ATOMIC(int32_t) state; + // Number of waiters both for read and write locks. + ATOMIC(int32_t) waiters; +#ifndef MC_ON + std::array padding = {0}; +#endif +}; + +__attribute__((visibility("default"))) void RWLockInit(struct rw_lock *rwlock); +__attribute__((visibility("default"))) void RWLockDestroy(struct rw_lock *rwlock); +__attribute__((visibility("default"))) ALWAYS_INLINE void ReadLock(struct rw_lock *rwlock); +__attribute__((visibility("default"))) void RWLockUnlock(struct rw_lock *rwlock); +__attribute__((visibility("default"))) void RWLockReadLock(struct rw_lock *rwlock); +__attribute__((visibility("default"))) void RWLockWriteLock(struct rw_lock *rwlock); +__attribute__((visibility("default"))) bool RWLockTryReadLock(struct rw_lock *rwlock); +__attribute__((visibility("default"))) bool RWLockTryWriteLock(struct rw_lock *rwlock); + #ifndef MC_ON } // namespace panda::os::unix::memory::futex #endif diff --git a/platforms/unix/libpandabase/futex/mutex.cpp b/platforms/unix/libpandabase/futex/mutex.cpp index 670d1d4ca..3f6216700 100644 --- a/platforms/unix/libpandabase/futex/mutex.cpp +++ b/platforms/unix/libpandabase/futex/mutex.cpp @@ -33,39 +33,6 @@ void PostFork() current_tid = os::thread::GetCurrentThreadId(); } -// Spin for small arguments and yield for longer ones. -static void BackOff(uint32_t i) -{ - static constexpr uint32_t SPIN_MAX = 10; - if (i <= SPIN_MAX) { - volatile uint32_t x = 0; // Volatile to make sure loop is not optimized out. - const uint32_t spin_count = 10 * i; - for (uint32_t spin = 0; spin < spin_count; spin++) { - ++x; - } - } else { - thread::Yield(); - } -} - -// Wait until pred is true, or until timeout is reached. -// Return true if the predicate test succeeded, false if we timed out. -template -static inline bool WaitBrieflyFor(std::atomic_int *addr, Pred pred) -{ - // We probably don't want to do syscall (switch context) when we use WaitBrieflyFor - static constexpr uint32_t MAX_BACK_OFF = 10; - static constexpr uint32_t MAX_ITER = 50; - for (uint32_t i = 1; i <= MAX_ITER; i++) { - BackOff(std::min(i, MAX_BACK_OFF)); - // Atomic with relaxed order reason: mutex synchronization - if (pred(addr->load(std::memory_order_relaxed))) { - return true; - } - } - return false; -} - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init) Mutex::Mutex() { @@ -107,172 +74,29 @@ void Mutex::UnlockForOther(thread::ThreadId thread) MutexUnlockForOther(&mutex_, thread); } -RWLock::~RWLock() +// NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init) +RWLock::RWLock() { -#ifndef PANDA_TARGET_MOBILE - if (!Mutex::DoNotCheckOnTerminationLoop()) { -#endif // PANDA_TARGET_MOBILE - // Atomic with relaxed order reason: mutex synchronization - if (state_.load(std::memory_order_relaxed) != 0) { - LOG(FATAL, COMMON) << "RWLock destruction failed; state_ is non zero!"; - // Atomic with relaxed order reason: mutex synchronization - } else if (exclusive_owner_.load(std::memory_order_relaxed) != 0) { - LOG(FATAL, COMMON) << "RWLock destruction failed; RWLock has an owner!"; - // Atomic with relaxed order reason: mutex synchronization - } else if (waiters_.load(std::memory_order_relaxed) != 0) { - LOG(FATAL, COMMON) << "RWLock destruction failed; RWLock has waiters!"; - } -#ifndef PANDA_TARGET_MOBILE - } else { - LOG(WARNING, COMMON) << "Termination loop detected, ignoring RWLock"; - } -#endif // PANDA_TARGET_MOBILE + RWLockInit(&rwlock_); } -void RWLock::WriteLock() +RWLock::~RWLock() { - if (current_tid == 0) { - current_tid = os::thread::GetCurrentThreadId(); - } - bool done = false; - while (!done) { - // Atomic with relaxed order reason: mutex synchronization - auto cur_state = state_.load(std::memory_order_relaxed); - if (LIKELY(cur_state == UNLOCKED)) { - // Unlocked, can acquire writelock - // Do CAS in case other thread beats us and acquires readlock first - done = state_.compare_exchange_weak(cur_state, WRITE_LOCKED, std::memory_order_acquire); - } else { - // Wait until RWLock is unlocked - if (!WaitBrieflyFor(&state_, [](int32_t state) { return state == UNLOCKED; })) { - // WaitBrieflyFor failed, go to futex wait - // Increment waiters count. - IncrementWaiters(); - // Retry wait until lock not held. If we have more than one reader, cur_state check fail - // doesn't mean this lock is unlocked. - while (cur_state != UNLOCKED) { - // NOLINTNEXTLINE(hicpp-signed-bitwise) - if (futex(GetStateAddr(), FUTEX_WAIT_PRIVATE, cur_state, nullptr, nullptr, 0) != 0) { - if ((errno != EAGAIN) && (errno != EINTR)) { - LOG(FATAL, COMMON) << "Futex wait failed!"; - } - } - // Atomic with relaxed order reason: mutex synchronization - cur_state = state_.load(std::memory_order_relaxed); - } - DecrementWaiters(); - } - } - } - // RWLock is held now - // Atomic with relaxed order reason: mutex synchronization - ASSERT(state_.load(std::memory_order_relaxed) == WRITE_LOCKED); - // Atomic with relaxed order reason: mutex synchronization - ASSERT(exclusive_owner_.load(std::memory_order_relaxed) == 0); - // Atomic with relaxed order reason: mutex synchronization - exclusive_owner_.store(current_tid, std::memory_order_relaxed); + RWLockDestroy(&rwlock_); } -void RWLock::HandleReadLockWait(int32_t cur_state) +void RWLock::WriteLock() { - // Wait until RWLock WriteLock is unlocked - if (!WaitBrieflyFor(&state_, [](int32_t state) { return state >= UNLOCKED; })) { - // WaitBrieflyFor failed, go to futex wait - IncrementWaiters(); - // Retry wait until WriteLock not held. - while (cur_state == WRITE_LOCKED) { - // NOLINTNEXTLINE(hicpp-signed-bitwise) - if (futex(GetStateAddr(), FUTEX_WAIT_PRIVATE, cur_state, nullptr, nullptr, 0) != 0) { - if ((errno != EAGAIN) && (errno != EINTR)) { - LOG(FATAL, COMMON) << "Futex wait failed!"; - } - } - // Atomic with relaxed order reason: mutex synchronization - cur_state = state_.load(std::memory_order_relaxed); - } - DecrementWaiters(); - } + return RWLockWriteLock(&rwlock_); } bool RWLock::TryReadLock() { - bool done = false; - // Atomic with relaxed order reason: mutex synchronization - auto cur_state = state_.load(std::memory_order_relaxed); - while (!done) { - if (cur_state >= UNLOCKED) { - auto new_state = cur_state + READ_INCREMENT; - // cur_state should be updated with fetched value on fail - done = state_.compare_exchange_weak(cur_state, new_state, std::memory_order_acquire); - } else { - // RWLock is Write held, trylock failed. - return false; - } - } - ASSERT(!HasExclusiveHolder()); - return true; + return RWLockTryReadLock(&rwlock_); } bool RWLock::TryWriteLock() { - if (current_tid == 0) { - current_tid = os::thread::GetCurrentThreadId(); - } - bool done = false; - // Atomic with relaxed order reason: mutex synchronization - auto cur_state = state_.load(std::memory_order_relaxed); - while (!done) { - if (LIKELY(cur_state == UNLOCKED)) { - // Unlocked, can acquire writelock - // Do CAS in case other thread beats us and acquires readlock first - // cur_state should be updated with fetched value on fail - done = state_.compare_exchange_weak(cur_state, WRITE_LOCKED, std::memory_order_acquire); - } else { - // RWLock is held, trylock failed. - return false; - } - } - // RWLock is held now - // Atomic with relaxed order reason: mutex synchronization - ASSERT(state_.load(std::memory_order_relaxed) == WRITE_LOCKED); - // Atomic with relaxed order reason: mutex synchronization - ASSERT(exclusive_owner_.load(std::memory_order_relaxed) == 0); - // Atomic with relaxed order reason: mutex synchronization - exclusive_owner_.store(current_tid, std::memory_order_relaxed); - return true; -} - -void RWLock::WriteUnlock() -{ - if (current_tid == 0) { - current_tid = os::thread::GetCurrentThreadId(); - } - ASSERT(IsExclusiveHeld(current_tid)); - - bool done = false; - // Atomic with relaxed order reason: mutex synchronization - int32_t cur_state = state_.load(std::memory_order_relaxed); - // CAS is weak and might fail, do in loop - while (!done) { - if (LIKELY(cur_state == WRITE_LOCKED)) { - // Reset exclusive owner before changing state to avoid check failures if other thread sees UNLOCKED - // Atomic with relaxed order reason: mutex synchronization - exclusive_owner_.store(0, std::memory_order_relaxed); - // Change state to unlocked and do release store. - // waiters_ load should not be reordered before state_, so it's done with seq cst. - // cur_state should be updated with fetched value on fail - done = state_.compare_exchange_weak(cur_state, UNLOCKED, std::memory_order_seq_cst); - if (LIKELY(done)) { - // We are doing write unlock, all waiters could be ReadLocks so we need to wake all. - // Atomic with seq_cst order reason: mutex synchronization - if (waiters_.load(std::memory_order_seq_cst) > 0) { - // NOLINTNEXTLINE(hicpp-signed-bitwise) - futex(GetStateAddr(), FUTEX_WAKE_PRIVATE, WAKE_ALL, nullptr, nullptr, 0); - } - } - } else { - LOG(FATAL, COMMON) << "RWLock WriteUnlock got unexpected state, RWLock is not writelocked?"; - } - } + return RWLockTryWriteLock(&rwlock_); } } // namespace panda::os::unix::memory::futex diff --git a/platforms/unix/libpandabase/futex/mutex.h b/platforms/unix/libpandabase/futex/mutex.h index f35148912..e15798a01 100644 --- a/platforms/unix/libpandabase/futex/mutex.h +++ b/platforms/unix/libpandabase/futex/mutex.h @@ -135,34 +135,19 @@ private: class SHARED_CAPABILITY("mutex") RWLock { public: - RWLock() = default; + PANDA_PUBLIC_API RWLock(); PANDA_PUBLIC_API ~RWLock(); // ReadLock and ReadUnlock are used in mutator lock often, prefer inlining over call to libpandabase ALWAYS_INLINE void ReadLock() ACQUIRE_SHARED() { - bool done = false; - while (!done) { - // Atomic with relaxed order reason: mutex synchronization - auto cur_state = state_.load(std::memory_order_relaxed); - if (LIKELY(cur_state >= UNLOCKED)) { - auto new_state = cur_state + READ_INCREMENT; - done = state_.compare_exchange_weak(cur_state, new_state, std::memory_order_acquire); - } else { - HandleReadLockWait(cur_state); - } - } - ASSERT(!HasExclusiveHolder()); + futex::RWLockReadLock(&rwlock_); } ALWAYS_INLINE void Unlock() RELEASE_GENERIC() { - if (HasExclusiveHolder()) { - WriteUnlock(); - } else { - ReadUnlock(); - } + futex::RWLockUnlock(&rwlock_); } PANDA_PUBLIC_API void WriteLock() ACQUIRE(); @@ -172,89 +157,8 @@ public: PANDA_PUBLIC_API bool TryWriteLock() TRY_ACQUIRE(true); private: - ALWAYS_INLINE void ReadUnlock() RELEASE_SHARED() - { - ASSERT(!HasExclusiveHolder()); - bool done = false; - // Atomic with relaxed order reason: mutex synchronization - auto cur_state = state_.load(std::memory_order_relaxed); - while (!done) { - if (LIKELY(cur_state > 0)) { - // Reduce state by 1 and do release store. - // waiters_ load should not be reordered before state_, so it's done with seq cst. - auto new_state = cur_state - READ_INCREMENT; - // cur_state should be updated with fetched value on fail - // Atomic with seq_cst order reason: mutex synchronization - done = state_.compare_exchange_weak(cur_state, new_state, std::memory_order_seq_cst); - if (done && new_state == UNLOCKED) { - // Atomic with seq_cst order reason: mutex synchronization - if (waiters_.load(std::memory_order_seq_cst) > 0) { - // Wake one exclusive waiter as there are now no readers. - // NOLINTNEXTLINE(hicpp-signed-bitwise) - futex(GetStateAddr(), FUTEX_WAKE_PRIVATE, WAKE_ALL, nullptr, nullptr, 0); - } - } - } else { - // Cannot use logger in header - std::cout << "RWLock ReadUnlock got unexpected state, RWLock is unlocked?" << std::endl; - std::abort(); - } - } - } - - PANDA_PUBLIC_API void WriteUnlock() RELEASE(); - - // Non-inline path for handling waiting. - PANDA_PUBLIC_API void HandleReadLockWait(int32_t cur_state); - - static constexpr int32_t WRITE_LOCKED = -1; - static constexpr int32_t UNLOCKED = 0; - static constexpr int32_t READ_INCREMENT = 1; - // -1 - write locked; 0 - unlocked; > 0 - read locked by state_ owners. - std::atomic_int32_t state_ {0}; - - int *GetStateAddr() - { - return reinterpret_cast(&state_); - } - - // Exclusive owner. - alignas(alignof(uint32_t)) std::atomic exclusive_owner_ {0}; static_assert(std::atomic::is_always_lock_free); - - bool HasExclusiveHolder() - { - // Atomic with relaxed order reason: mutex synchronization - return exclusive_owner_.load(std::memory_order_relaxed) != 0; - } - bool IsExclusiveHeld(thread::ThreadId thread) - { - // Atomic with relaxed order reason: mutex synchronization - return exclusive_owner_.load(std::memory_order_relaxed) == thread; - } - - // Number of waiters both for read and write locks. - std::atomic_uint32_t waiters_ {0}; - - void IncrementWaiters() - { - // Atomic with relaxed order reason: mutex synchronization - waiters_.fetch_add(1, std::memory_order_relaxed); - } - void DecrementWaiters() - { - // Atomic with relaxed order reason: mutex synchronization - waiters_.fetch_sub(1, std::memory_order_relaxed); - } - - // Extra padding to make RWLock 16 bytes long - static constexpr size_t PADDING_SIZE = 1; - std::array padding_ = {0}; - // [[maybe_unused]] causes issues, dummy accessor for `padding_` as workaround - uint32_t DummyAccessPadding() - { - return padding_[0]; - } + struct rw_lock rwlock_; NO_COPY_SEMANTIC(RWLock); NO_MOVE_SEMANTIC(RWLock); @@ -302,7 +206,6 @@ private: static constexpr size_t ALL_STRUCTURES_SIZE = 16U; static_assert(sizeof(ConditionVariable) == ALL_STRUCTURES_SIZE); -static_assert(sizeof(RWLock) == ALL_STRUCTURES_SIZE); } // namespace panda::os::unix::memory::futex #endif // PANDA_LIBPANDABASE_PBASE_OS_UNIX__FUTEX_MUTEX_H_ -- Gitee From 246baa4de93a2d99b8a1e38e7529d3b3eb9b5e5c Mon Sep 17 00:00:00 2001 From: Pavel Andrianov Date: Tue, 7 Feb 2023 17:48:51 +0300 Subject: [PATCH 02/17] Enable GenMC tests Change-Id: I5d013f7e4206abbb035463511084e7f208da6e4a --- libpandabase/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libpandabase/CMakeLists.txt b/libpandabase/CMakeLists.txt index 5e7f71192..31c06d2d2 100644 --- a/libpandabase/CMakeLists.txt +++ b/libpandabase/CMakeLists.txt @@ -415,10 +415,10 @@ if (DEFINED ENV{GENMC_PATH}) ${PANDA_ROOT}/libpandabase/tests/genmc/mutex_test_2.cpp ${PANDA_ROOT}/libpandabase/tests/genmc/mutex_test_3.cpp ${PANDA_ROOT}/libpandabase/tests/genmc/mutex_test_4.cpp - # ${PANDA_ROOT}/libpandabase/tests/genmc/condvar_test_1.cpp + ${PANDA_ROOT}/libpandabase/tests/genmc/condvar_test_1.cpp # ${PANDA_ROOT}/libpandabase/tests/genmc/condvar_test_2.cpp - # ${PANDA_ROOT}/libpandabase/tests/genmc/condvar_test_3.cpp - # ${PANDA_ROOT}/libpandabase/tests/genmc/rwlock_test_1.cpp + ${PANDA_ROOT}/libpandabase/tests/genmc/condvar_test_3.cpp + ${PANDA_ROOT}/libpandabase/tests/genmc/rwlock_test_1.cpp ) foreach(genmc_test ${GENMC_TESTS}) -- Gitee From 04ad4f7dff552fdfcc412a1a0824956290bb1dd1 Mon Sep 17 00:00:00 2001 From: Pavel Andrianov Date: Wed, 8 Feb 2023 09:41:20 +0300 Subject: [PATCH 03/17] Replace genmc.h to decl Change-Id: I0c919b11427f7330667315cc66b97deb7bc17e56 --- libpandabase/tests/genmc/condvar_test_1.cpp | 2 +- libpandabase/tests/genmc/condvar_test_2.cpp | 2 +- libpandabase/tests/genmc/condvar_test_3.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libpandabase/tests/genmc/condvar_test_1.cpp b/libpandabase/tests/genmc/condvar_test_1.cpp index cf6454e14..e1e977d00 100644 --- a/libpandabase/tests/genmc/condvar_test_1.cpp +++ b/libpandabase/tests/genmc/condvar_test_1.cpp @@ -14,7 +14,6 @@ */ #include "common.h" -#include "genmc.h" // The test checks the work of Wait-SignalOne // Thread1 sets g_shared to 1, then waits, @@ -27,6 +26,7 @@ extern "C" void __VERIFIER_assume(int) __attribute__((__nothrow__)); static struct fmutex g_x; static struct CondVar g_c; constexpr int N = 2; +extern "C" void __VERIFIER_assume(int) __attribute__ ((__nothrow__)); static void *Thread1(void *arg) { diff --git a/libpandabase/tests/genmc/condvar_test_2.cpp b/libpandabase/tests/genmc/condvar_test_2.cpp index 5bdbe0450..ab406c99c 100644 --- a/libpandabase/tests/genmc/condvar_test_2.cpp +++ b/libpandabase/tests/genmc/condvar_test_2.cpp @@ -14,7 +14,6 @@ */ #include "common.h" -#include "genmc.h" // The test checks the work of Wait-SignalAll // There are two waiters (Thread1) and one waker (Thread2) @@ -28,6 +27,7 @@ extern "C" void __VERIFIER_assume(int) __attribute__((__nothrow__)); static struct fmutex g_x; static struct CondVar g_c; constexpr int N = 2; +extern "C" void __VERIFIER_assume(int) __attribute__ ((__nothrow__)); static void *Thread1(void *arg) { diff --git a/libpandabase/tests/genmc/condvar_test_3.cpp b/libpandabase/tests/genmc/condvar_test_3.cpp index 0ca732099..932914580 100644 --- a/libpandabase/tests/genmc/condvar_test_3.cpp +++ b/libpandabase/tests/genmc/condvar_test_3.cpp @@ -14,7 +14,6 @@ */ #include "common.h" -#include "genmc.h" // The test checks the work of TimedWait-SignalOne // Thread1 sets g_shared to 1, then waits with timeout, @@ -27,6 +26,7 @@ extern "C" void __VERIFIER_assume(int) __attribute__((__nothrow__)); static struct fmutex g_x; static struct CondVar g_c; constexpr int N = 2; +extern "C" void __VERIFIER_assume(int) __attribute__ ((__nothrow__)); static void *Thread1(void *arg) { -- Gitee From 362707aa7e543406983f8408a6e487dadc3a9c20 Mon Sep 17 00:00:00 2001 From: Pavel Andrianov Date: Fri, 14 Apr 2023 11:02:15 +0300 Subject: [PATCH 04/17] Disable the test Change-Id: I2afbdb9f22f0e74afcb4c22d7433119d3d9251c0 --- libpandabase/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libpandabase/CMakeLists.txt b/libpandabase/CMakeLists.txt index 31c06d2d2..0fe4f9733 100644 --- a/libpandabase/CMakeLists.txt +++ b/libpandabase/CMakeLists.txt @@ -416,9 +416,10 @@ if (DEFINED ENV{GENMC_PATH}) ${PANDA_ROOT}/libpandabase/tests/genmc/mutex_test_3.cpp ${PANDA_ROOT}/libpandabase/tests/genmc/mutex_test_4.cpp ${PANDA_ROOT}/libpandabase/tests/genmc/condvar_test_1.cpp + # Takes too much time #8866 # ${PANDA_ROOT}/libpandabase/tests/genmc/condvar_test_2.cpp ${PANDA_ROOT}/libpandabase/tests/genmc/condvar_test_3.cpp - ${PANDA_ROOT}/libpandabase/tests/genmc/rwlock_test_1.cpp + # ${PANDA_ROOT}/libpandabase/tests/genmc/rwlock_test_1.cpp ) foreach(genmc_test ${GENMC_TESTS}) -- Gitee From c46dd13f90453c85ca5cbe6236fa509da872a170 Mon Sep 17 00:00:00 2001 From: Pavel Andrianov Date: Mon, 17 Apr 2023 10:10:29 +0300 Subject: [PATCH 05/17] Add a couple more tests Change-Id: Ifc40339b8093bc67c28b7828df9e8abbc4bc8038 --- libpandabase/CMakeLists.txt | 5 +- libpandabase/tests/genmc/rwlock_test_1.cpp | 3 - libpandabase/tests/genmc/rwlock_test_2.cpp | 68 ++++++++++++++++++++++ libpandabase/tests/genmc/rwlock_test_3.cpp | 65 +++++++++++++++++++++ 4 files changed, 137 insertions(+), 4 deletions(-) create mode 100644 libpandabase/tests/genmc/rwlock_test_2.cpp create mode 100644 libpandabase/tests/genmc/rwlock_test_3.cpp diff --git a/libpandabase/CMakeLists.txt b/libpandabase/CMakeLists.txt index 0fe4f9733..8c0d8acb1 100644 --- a/libpandabase/CMakeLists.txt +++ b/libpandabase/CMakeLists.txt @@ -419,7 +419,10 @@ if (DEFINED ENV{GENMC_PATH}) # Takes too much time #8866 # ${PANDA_ROOT}/libpandabase/tests/genmc/condvar_test_2.cpp ${PANDA_ROOT}/libpandabase/tests/genmc/condvar_test_3.cpp - # ${PANDA_ROOT}/libpandabase/tests/genmc/rwlock_test_1.cpp + ${PANDA_ROOT}/libpandabase/tests/genmc/rwlock_test_1.cpp + # Takes too much time + # ${PANDA_ROOT}/libpandabase/tests/genmc/rwlock_test_2.cpp + ${PANDA_ROOT}/libpandabase/tests/genmc/rwlock_test_3.cpp ) foreach(genmc_test ${GENMC_TESTS}) diff --git a/libpandabase/tests/genmc/rwlock_test_1.cpp b/libpandabase/tests/genmc/rwlock_test_1.cpp index 64bc9aa44..6addc38d7 100644 --- a/libpandabase/tests/genmc/rwlock_test_1.cpp +++ b/libpandabase/tests/genmc/rwlock_test_1.cpp @@ -50,17 +50,14 @@ int main() pthread_t t1; pthread_t t2; pthread_t t3; - pthread_t t4; pthread_create(&t1, nullptr, Reader, nullptr); pthread_create(&t2, nullptr, Reader, nullptr); pthread_create(&t3, nullptr, Writer, nullptr); - pthread_create(&t4, nullptr, Writer, nullptr); pthread_join(t1, nullptr); pthread_join(t2, nullptr); pthread_join(t3, nullptr); - pthread_join(t4, nullptr); // Check that writer was finished ASSERT(g_shared == 2); diff --git a/libpandabase/tests/genmc/rwlock_test_2.cpp b/libpandabase/tests/genmc/rwlock_test_2.cpp new file mode 100644 index 000000000..fa5c2365a --- /dev/null +++ b/libpandabase/tests/genmc/rwlock_test_2.cpp @@ -0,0 +1,68 @@ +/** + * Copyright (c) 2021-2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common.h" + +// The test checks basic work of RWLock +// Reader reads the global variable, then checks its value +// Writer increments the global variable and checks its value + +static struct rw_lock lock; + +static void *Reader(void *arg) +{ + RWLockReadLock(&lock); + int tmp = g_shared; + ASSERT(tmp == g_shared); + RWLockUnlock(&lock); + + ASSERT(tmp == 0 || tmp == 1 || tmp == 2); + return nullptr; +} + +static void *Writer(void *arg) +{ + RWLockWriteLock(&lock); + // increment global var and check its value + CheckGlobalVar(g_shared + 1); + RWLockUnlock(&lock); + return nullptr; +} + +int main() +{ + RWLockInit(&lock); + g_shared = 0; + pthread_t t1; + pthread_t t2; + pthread_t t3; + pthread_t t4; + + pthread_create(&t1, nullptr, Reader, nullptr); + pthread_create(&t2, nullptr, Reader, nullptr); + pthread_create(&t3, nullptr, Writer, nullptr); + pthread_create(&t4, nullptr, Writer, nullptr); + + pthread_join(t1, nullptr); + pthread_join(t2, nullptr); + pthread_join(t3, nullptr); + pthread_join(t4, nullptr); + + // Check that writer was finished + ASSERT(g_shared == 2); + + RWLockDestroy(&lock); + return 0; +} \ No newline at end of file diff --git a/libpandabase/tests/genmc/rwlock_test_3.cpp b/libpandabase/tests/genmc/rwlock_test_3.cpp new file mode 100644 index 000000000..6f05ebc77 --- /dev/null +++ b/libpandabase/tests/genmc/rwlock_test_3.cpp @@ -0,0 +1,65 @@ +/** + * Copyright (c) 2021-2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common.h" + +// The test checks basic work of RWLock +// Reader reads the global variable, then checks its value +// Writer increments the global variable and checks its value + +static struct rw_lock lock; + +static void *Reader(void *arg) +{ + RWLockReadLock(&lock); + int tmp = g_shared; + ASSERT(tmp == g_shared); + RWLockUnlock(&lock); + + ASSERT(tmp == 0 || tmp == 1 || tmp == 2); + return nullptr; +} + +static void *Writer(void *arg) +{ + RWLockWriteLock(&lock); + // increment global var and check its value + CheckGlobalVar(g_shared + 1); + RWLockUnlock(&lock); + return nullptr; +} + +int main() +{ + RWLockInit(&lock); + g_shared = 0; + pthread_t t1; + pthread_t t3; + pthread_t t4; + + pthread_create(&t1, nullptr, Reader, nullptr); + pthread_create(&t3, nullptr, Writer, nullptr); + pthread_create(&t4, nullptr, Writer, nullptr); + + pthread_join(t1, nullptr); + pthread_join(t3, nullptr); + pthread_join(t4, nullptr); + + // Check that writer was finished + ASSERT(g_shared == 2); + + RWLockDestroy(&lock); + return 0; +} \ No newline at end of file -- Gitee From 0e27989283d950b3c03633efba15ab66fc6a714e Mon Sep 17 00:00:00 2001 From: Pavel Andrianov Date: Mon, 17 Apr 2023 12:06:17 +0300 Subject: [PATCH 06/17] Refactoring Change-Id: Id9d333e82219e68094198f3607c4cd44c3efacb2 --- libpandabase/tests/genmc/condvar_test_1.cpp | 1 - libpandabase/tests/genmc/condvar_test_2.cpp | 1 - libpandabase/tests/genmc/condvar_test_3.cpp | 1 - platforms/unix/libpandabase/futex/fmutex.cpp | 26 +++++++++++--------- platforms/unix/libpandabase/futex/fmutex.h | 5 ---- 5 files changed, 15 insertions(+), 19 deletions(-) diff --git a/libpandabase/tests/genmc/condvar_test_1.cpp b/libpandabase/tests/genmc/condvar_test_1.cpp index e1e977d00..578af55e9 100644 --- a/libpandabase/tests/genmc/condvar_test_1.cpp +++ b/libpandabase/tests/genmc/condvar_test_1.cpp @@ -26,7 +26,6 @@ extern "C" void __VERIFIER_assume(int) __attribute__((__nothrow__)); static struct fmutex g_x; static struct CondVar g_c; constexpr int N = 2; -extern "C" void __VERIFIER_assume(int) __attribute__ ((__nothrow__)); static void *Thread1(void *arg) { diff --git a/libpandabase/tests/genmc/condvar_test_2.cpp b/libpandabase/tests/genmc/condvar_test_2.cpp index ab406c99c..73f7b159d 100644 --- a/libpandabase/tests/genmc/condvar_test_2.cpp +++ b/libpandabase/tests/genmc/condvar_test_2.cpp @@ -27,7 +27,6 @@ extern "C" void __VERIFIER_assume(int) __attribute__((__nothrow__)); static struct fmutex g_x; static struct CondVar g_c; constexpr int N = 2; -extern "C" void __VERIFIER_assume(int) __attribute__ ((__nothrow__)); static void *Thread1(void *arg) { diff --git a/libpandabase/tests/genmc/condvar_test_3.cpp b/libpandabase/tests/genmc/condvar_test_3.cpp index 932914580..e245da709 100644 --- a/libpandabase/tests/genmc/condvar_test_3.cpp +++ b/libpandabase/tests/genmc/condvar_test_3.cpp @@ -26,7 +26,6 @@ extern "C" void __VERIFIER_assume(int) __attribute__((__nothrow__)); static struct fmutex g_x; static struct CondVar g_c; constexpr int N = 2; -extern "C" void __VERIFIER_assume(int) __attribute__ ((__nothrow__)); static void *Thread1(void *arg) { diff --git a/platforms/unix/libpandabase/futex/fmutex.cpp b/platforms/unix/libpandabase/futex/fmutex.cpp index 7de8c6c34..edf342cc6 100644 --- a/platforms/unix/libpandabase/futex/fmutex.cpp +++ b/platforms/unix/libpandabase/futex/fmutex.cpp @@ -646,7 +646,7 @@ void RWLockReadLock(struct rw_lock *const rwlock) auto cur_state = ATOMIC_LOAD(&rwlock->state, memory_order_relaxed); if (LIKELY(cur_state != WRITE_LOCKED)) { auto new_state = cur_state + READ_INCREMENT; - done = ATOMIC_CAS_WEAK_1(&rwlock->state, cur_state, new_state, memory_order_acquire); + done = ATOMIC_CAS_WEAK(&rwlock->state, cur_state, new_state, memory_order_acquire, memory_order_relaxed); } else { HandleReadLockWait(rwlock, cur_state); } @@ -666,7 +666,7 @@ void RWLockWriteLock(struct rw_lock *const rwlock) if (LIKELY(cur_state == UNLOCKED)) { // Unlocked, can acquire writelock // Do CAS in case other thread beats us and acquires readlock first - done = ATOMIC_CAS_WEAK_1(&rwlock->state, cur_state, WRITE_LOCKED, memory_order_acquire); + done = ATOMIC_CAS_WEAK(&rwlock->state, cur_state, WRITE_LOCKED, memory_order_acquire, memory_order_relaxed); } else { // Wait until RWLock is unlocked if (!WaitBrieflyFor(&rwlock->state, false)) { @@ -707,7 +707,7 @@ bool RWLockTryReadLock(struct rw_lock *const rwlock) if (cur_state >= UNLOCKED) { auto new_state = cur_state + READ_INCREMENT; // cur_state should be updated with fetched value on fail - done = ATOMIC_CAS_WEAK_1(&rwlock->state, cur_state, new_state, memory_order_acquire); + done = ATOMIC_CAS_WEAK(&rwlock->state, cur_state, new_state, memory_order_acquire, memory_order_relaxed); } else { // RWLock is Write held, trylock failed. return false; @@ -730,7 +730,7 @@ bool RWLockTryWriteLock(struct rw_lock *const rwlock) // Unlocked, can acquire writelock // Do CAS in case other thread beats us and acquires readlock first // cur_state should be updated with fetched value on fail - done = ATOMIC_CAS_WEAK_1(&rwlock->state, cur_state, WRITE_LOCKED, memory_order_acquire); + done = ATOMIC_CAS_WEAK(&rwlock->state, cur_state, WRITE_LOCKED, memory_order_acquire, memory_order_relaxed); } else { // RWLock is held, trylock failed. return false; @@ -755,7 +755,7 @@ void RWLockReadUnlock(struct rw_lock *const rwlock) auto new_state = cur_state - READ_INCREMENT; // cur_state should be updated with fetched value on fail // Atomic with seq_cst order reason: mutex synchronization - done = ATOMIC_CAS_WEAK_1(&rwlock->state, cur_state, new_state, memory_order_release); + done = ATOMIC_CAS_WEAK(&rwlock->state, cur_state, new_state, memory_order_release, memory_order_relaxed); if (done && new_state == UNLOCKED) { // Atomic with seq_cst order reason: mutex synchronization if (ATOMIC_LOAD(&rwlock->waiters, memory_order_relaxed) > 0) { @@ -783,17 +783,21 @@ void RWLockWriteUnlock(struct rw_lock *const rwlock) bool done = false; // Atomic with relaxed order reason: mutex synchronization - int32_t cur_state = ATOMIC_LOAD(&rwlock->state, memory_order_relaxed); + auto cur_state = ATOMIC_LOAD(&rwlock->state, memory_order_relaxed); // CAS is weak and might fail, do in loop while (!done) { - if (LIKELY(cur_state == WRITE_LOCKED)) { + //if (LIKELY(cur_state == WRITE_LOCKED)) { // Change state to unlocked and do release store. // waiters load should not be reordered before state, so it's done with seq cst. // cur_state should be updated with fetched value on fail - done = ATOMIC_CAS_WEAK_1(&rwlock->state, cur_state, UNLOCKED, memory_order_release); - } else { - FAIL_WITH_MESSAGE("RWLock WriteUnlock got unexpected state, RWLock is not writelocked?"); - } + done = ATOMIC_CAS_WEAK(&rwlock->state, cur_state, UNLOCKED, memory_order_release, memory_order_relaxed); +#ifdef MC_ON + __VERIFIER_assume(done); +#endif + //ATOMIC_STORE(&rwlock->state, UNLOCKED, memory_order_relaxed); + // } else { + // FAIL_WITH_MESSAGE("RWLock WriteUnlock got unexpected state, RWLock is not writelocked?"); + // } } // We are doing write unlock, all waiters could be ReadLocks so we need to wake all. // Atomic with seq_cst order reason: mutex synchronization diff --git a/platforms/unix/libpandabase/futex/fmutex.h b/platforms/unix/libpandabase/futex/fmutex.h index 25f64102b..08648d851 100644 --- a/platforms/unix/libpandabase/futex/fmutex.h +++ b/platforms/unix/libpandabase/futex/fmutex.h @@ -31,8 +31,6 @@ #define ATOMIC_FETCH_SUB(addr, val, mem) atomic_fetch_sub_explicit(addr, val, mem) #define ATOMIC_CAS_WEAK(addr, old_val, new_val, mem1, mem2) \ atomic_compare_exchange_weak_explicit(addr, &old_val, new_val, mem1, mem2) -#define ATOMIC_CAS_WEAK_1(addr, old_val, new_val, mem) \ - atomic_compare_exchange_weak_explicit(addr, &old_val, new_val, mem, mem) #define ASSERT(a) assert(a) #define LIKELY(a) a #define UNLIKELY(a) a @@ -58,15 +56,12 @@ namespace panda::os::unix::memory::futex { // NOLINTNEXTLINE(cppcoreguidelines-macro-usage) #define ATOMIC_CAS_WEAK(addr, old_val, new_val, mem1, mem2) \ (addr)->compare_exchange_weak(old_val, new_val, std::mem1, std::mem2) -// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) -#define ATOMIC_CAS_WEAK_1(addr, old_val, new_val, mem) (addr)->compare_exchange_weak(old_val, new_val, std::mem) #endif // Copy of mutex storage, after complete implementation should totally replace mutex::current_tid // NOLINTNEXTLINE(readability-identifier-naming) extern thread_local THREAD_ID current_tid; -// Empty string void MutexInit(struct fmutex *m); void MutexDestroy(struct fmutex *m); bool MutexLock(struct fmutex *m, bool trylock); -- Gitee From f804353d64df0129b8dac9360d3d40f1f3e0b65f Mon Sep 17 00:00:00 2001 From: Pavel Andrianov Date: Mon, 17 Apr 2023 12:15:28 +0300 Subject: [PATCH 07/17] Add assumptions Change-Id: I31ab451bab8030f0f97c0cd226bae5bbfe6f0401 --- libpandabase/CMakeLists.txt | 3 +-- platforms/unix/libpandabase/futex/fmutex.cpp | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/libpandabase/CMakeLists.txt b/libpandabase/CMakeLists.txt index 8c0d8acb1..7b9e63410 100644 --- a/libpandabase/CMakeLists.txt +++ b/libpandabase/CMakeLists.txt @@ -420,8 +420,7 @@ if (DEFINED ENV{GENMC_PATH}) # ${PANDA_ROOT}/libpandabase/tests/genmc/condvar_test_2.cpp ${PANDA_ROOT}/libpandabase/tests/genmc/condvar_test_3.cpp ${PANDA_ROOT}/libpandabase/tests/genmc/rwlock_test_1.cpp - # Takes too much time - # ${PANDA_ROOT}/libpandabase/tests/genmc/rwlock_test_2.cpp + ${PANDA_ROOT}/libpandabase/tests/genmc/rwlock_test_2.cpp ${PANDA_ROOT}/libpandabase/tests/genmc/rwlock_test_3.cpp ) diff --git a/platforms/unix/libpandabase/futex/fmutex.cpp b/platforms/unix/libpandabase/futex/fmutex.cpp index edf342cc6..6cf2897cc 100644 --- a/platforms/unix/libpandabase/futex/fmutex.cpp +++ b/platforms/unix/libpandabase/futex/fmutex.cpp @@ -647,6 +647,9 @@ void RWLockReadLock(struct rw_lock *const rwlock) if (LIKELY(cur_state != WRITE_LOCKED)) { auto new_state = cur_state + READ_INCREMENT; done = ATOMIC_CAS_WEAK(&rwlock->state, cur_state, new_state, memory_order_acquire, memory_order_relaxed); +#ifdef MC_ON + __VERIFIER_assume(done); +#endif } else { HandleReadLockWait(rwlock, cur_state); } @@ -667,6 +670,9 @@ void RWLockWriteLock(struct rw_lock *const rwlock) // Unlocked, can acquire writelock // Do CAS in case other thread beats us and acquires readlock first done = ATOMIC_CAS_WEAK(&rwlock->state, cur_state, WRITE_LOCKED, memory_order_acquire, memory_order_relaxed); +#ifdef MC_ON + __VERIFIER_assume(done); +#endif } else { // Wait until RWLock is unlocked if (!WaitBrieflyFor(&rwlock->state, false)) { @@ -708,6 +714,9 @@ bool RWLockTryReadLock(struct rw_lock *const rwlock) auto new_state = cur_state + READ_INCREMENT; // cur_state should be updated with fetched value on fail done = ATOMIC_CAS_WEAK(&rwlock->state, cur_state, new_state, memory_order_acquire, memory_order_relaxed); +#ifdef MC_ON + __VERIFIER_assume(done); +#endif } else { // RWLock is Write held, trylock failed. return false; @@ -731,6 +740,9 @@ bool RWLockTryWriteLock(struct rw_lock *const rwlock) // Do CAS in case other thread beats us and acquires readlock first // cur_state should be updated with fetched value on fail done = ATOMIC_CAS_WEAK(&rwlock->state, cur_state, WRITE_LOCKED, memory_order_acquire, memory_order_relaxed); +#ifdef MC_ON + __VERIFIER_assume(done); +#endif } else { // RWLock is held, trylock failed. return false; @@ -756,6 +768,9 @@ void RWLockReadUnlock(struct rw_lock *const rwlock) // cur_state should be updated with fetched value on fail // Atomic with seq_cst order reason: mutex synchronization done = ATOMIC_CAS_WEAK(&rwlock->state, cur_state, new_state, memory_order_release, memory_order_relaxed); +#ifdef MC_ON + __VERIFIER_assume(done); +#endif if (done && new_state == UNLOCKED) { // Atomic with seq_cst order reason: mutex synchronization if (ATOMIC_LOAD(&rwlock->waiters, memory_order_relaxed) > 0) { @@ -776,9 +791,7 @@ void RWLockReadUnlock(struct rw_lock *const rwlock) void RWLockWriteUnlock(struct rw_lock *const rwlock) { - if (current_tid == 0) { - current_tid = GET_CURRENT_THREAD; - } + ASSERT(current_tid != 0); ASSERT(HasExclusiveHolder(rwlock)); bool done = false; -- Gitee From 475d5234cf38b35a9ae88b8e339df9ae66e6c02f Mon Sep 17 00:00:00 2001 From: Pavel Andrianov Date: Tue, 18 Apr 2023 10:57:39 +0300 Subject: [PATCH 08/17] Remove debug code Change-Id: I237eedbfa54eb332e394124a81f0b6c38a1a8ce1 --- platforms/unix/libpandabase/futex/fmutex.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/platforms/unix/libpandabase/futex/fmutex.cpp b/platforms/unix/libpandabase/futex/fmutex.cpp index 6cf2897cc..fdfdc6ea7 100644 --- a/platforms/unix/libpandabase/futex/fmutex.cpp +++ b/platforms/unix/libpandabase/futex/fmutex.cpp @@ -799,7 +799,7 @@ void RWLockWriteUnlock(struct rw_lock *const rwlock) auto cur_state = ATOMIC_LOAD(&rwlock->state, memory_order_relaxed); // CAS is weak and might fail, do in loop while (!done) { - //if (LIKELY(cur_state == WRITE_LOCKED)) { + if (LIKELY(cur_state == WRITE_LOCKED)) { // Change state to unlocked and do release store. // waiters load should not be reordered before state, so it's done with seq cst. // cur_state should be updated with fetched value on fail @@ -807,10 +807,9 @@ void RWLockWriteUnlock(struct rw_lock *const rwlock) #ifdef MC_ON __VERIFIER_assume(done); #endif - //ATOMIC_STORE(&rwlock->state, UNLOCKED, memory_order_relaxed); - // } else { - // FAIL_WITH_MESSAGE("RWLock WriteUnlock got unexpected state, RWLock is not writelocked?"); - // } + } else { + FAIL_WITH_MESSAGE("RWLock WriteUnlock got unexpected state, RWLock is not writelocked?"); + } } // We are doing write unlock, all waiters could be ReadLocks so we need to wake all. // Atomic with seq_cst order reason: mutex synchronization -- Gitee From aff90f3b22c34fda8358c42ada8aa19f68ded3ac Mon Sep 17 00:00:00 2001 From: Pavel Andrianov Date: Tue, 18 Apr 2023 13:35:26 +0300 Subject: [PATCH 09/17] Fix code style Change-Id: I507cfe4064d576b74534eb3be4f2e2b5ba02ac29 --- platforms/unix/libpandabase/futex/fmutex.cpp | 28 ++++++++++---------- platforms/unix/libpandabase/futex/fmutex.h | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/platforms/unix/libpandabase/futex/fmutex.cpp b/platforms/unix/libpandabase/futex/fmutex.cpp index fdfdc6ea7..8b116f65b 100644 --- a/platforms/unix/libpandabase/futex/fmutex.cpp +++ b/platforms/unix/libpandabase/futex/fmutex.cpp @@ -566,35 +566,35 @@ void SignalCount(struct CondVar *const cond, int32_t to_wake) } // Empty string -int *GetStateAddr(struct rw_lock *const rwlock) +int *GetStateAddr(struct RwLock *const rwlock) { return reinterpret_cast(&rwlock->state); } -bool HasExclusiveHolder(struct rw_lock *const rwlock) +bool HasExclusiveHolder(struct RwLock *const rwlock) { // Atomic with relaxed order reason: mutex synchronization return ATOMIC_LOAD(&rwlock->state, memory_order_relaxed) == WRITE_LOCKED; } -void IncrementWaiters(struct rw_lock *const rwlock) +void IncrementWaiters(struct RwLock *const rwlock) { // Atomic with relaxed order reason: mutex synchronization ATOMIC_FETCH_ADD(&rwlock->waiters, 1, memory_order_relaxed); } -void DecrementWaiters(struct rw_lock *const rwlock) +void DecrementWaiters(struct RwLock *const rwlock) { // Atomic with relaxed order reason: mutex synchronization ATOMIC_FETCH_SUB(&rwlock->waiters, 1, memory_order_relaxed); } -void RWLockInit(struct rw_lock *const rwlock) +void RWLockInit(struct RwLock *const rwlock) { ATOMIC_STORE(&rwlock->state, 0, memory_order_relaxed); ATOMIC_STORE(&rwlock->waiters, 0, memory_order_relaxed); } -void RWLockDestroy(struct rw_lock *const rwlock) +void RWLockDestroy(struct RwLock *const rwlock) { #ifndef PANDA_TARGET_MOBILE if (!MutexDoNotCheckOnTerminationLoop()) { @@ -613,7 +613,7 @@ void RWLockDestroy(struct rw_lock *const rwlock) #endif // PANDA_TARGET_MOBILE } -void HandleReadLockWait(struct rw_lock *const rwlock, int32_t cur_state) +void HandleReadLockWait(struct RwLock *const rwlock, int32_t cur_state) { // Wait until RWLock WriteLock is unlocked if (!WaitBrieflyFor(&rwlock->state, true)) { @@ -638,7 +638,7 @@ void HandleReadLockWait(struct rw_lock *const rwlock, int32_t cur_state) } } -void RWLockReadLock(struct rw_lock *const rwlock) +void RWLockReadLock(struct RwLock *const rwlock) { bool done = false; do { @@ -657,7 +657,7 @@ void RWLockReadLock(struct rw_lock *const rwlock) ASSERT(!HasExclusiveHolder(rwlock)); } -void RWLockWriteLock(struct rw_lock *const rwlock) +void RWLockWriteLock(struct RwLock *const rwlock) { if (current_tid == 0) { current_tid = GET_CURRENT_THREAD; @@ -704,7 +704,7 @@ void RWLockWriteLock(struct rw_lock *const rwlock) ASSERT(ATOMIC_LOAD(&rwlock->state, memory_order_relaxed) == WRITE_LOCKED); } -bool RWLockTryReadLock(struct rw_lock *const rwlock) +bool RWLockTryReadLock(struct RwLock *const rwlock) { bool done = false; // Atomic with relaxed order reason: mutex synchronization @@ -726,7 +726,7 @@ bool RWLockTryReadLock(struct rw_lock *const rwlock) return true; } -bool RWLockTryWriteLock(struct rw_lock *const rwlock) +bool RWLockTryWriteLock(struct RwLock *const rwlock) { if (current_tid == 0) { current_tid = GET_CURRENT_THREAD; @@ -754,7 +754,7 @@ bool RWLockTryWriteLock(struct rw_lock *const rwlock) return true; } -void RWLockReadUnlock(struct rw_lock *const rwlock) +void RWLockReadUnlock(struct RwLock *const rwlock) { ASSERT(!HasExclusiveHolder(rwlock)); bool done = false; @@ -789,7 +789,7 @@ void RWLockReadUnlock(struct rw_lock *const rwlock) } while (!done); } -void RWLockWriteUnlock(struct rw_lock *const rwlock) +void RWLockWriteUnlock(struct RwLock *const rwlock) { ASSERT(current_tid != 0); ASSERT(HasExclusiveHolder(rwlock)); @@ -823,7 +823,7 @@ void RWLockWriteUnlock(struct rw_lock *const rwlock) } } -void RWLockUnlock(struct rw_lock *const rwlock) +void RWLockUnlock(struct RwLock *const rwlock) { if (HasExclusiveHolder(rwlock)) { RWLockWriteUnlock(rwlock); diff --git a/platforms/unix/libpandabase/futex/fmutex.h b/platforms/unix/libpandabase/futex/fmutex.h index 08648d851..6d85c7cfe 100644 --- a/platforms/unix/libpandabase/futex/fmutex.h +++ b/platforms/unix/libpandabase/futex/fmutex.h @@ -132,7 +132,7 @@ static constexpr int32_t READ_INCREMENT = 1; // Extra padding to make RWLock 16 bytes long static constexpr size_t PADDING_SIZE = 1; -struct rw_lock { +struct RwLock { // -1 - write locked; 0 - unlocked; > 0 - read locked by state_ owners. ATOMIC(int32_t) state; // Number of waiters both for read and write locks. -- Gitee From d77704ff3e99c18cfb345849992ffac8f6e72352 Mon Sep 17 00:00:00 2001 From: Pavel Andrianov Date: Tue, 18 Apr 2023 14:02:17 +0300 Subject: [PATCH 10/17] Missed rename Change-Id: I218c321065def8d2b25b0fd996d0c0dabe900f27 --- platforms/unix/libpandabase/futex/fmutex.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/platforms/unix/libpandabase/futex/fmutex.h b/platforms/unix/libpandabase/futex/fmutex.h index 6d85c7cfe..fc7319f48 100644 --- a/platforms/unix/libpandabase/futex/fmutex.h +++ b/platforms/unix/libpandabase/futex/fmutex.h @@ -142,14 +142,14 @@ struct RwLock { #endif }; -__attribute__((visibility("default"))) void RWLockInit(struct rw_lock *rwlock); -__attribute__((visibility("default"))) void RWLockDestroy(struct rw_lock *rwlock); -__attribute__((visibility("default"))) ALWAYS_INLINE void ReadLock(struct rw_lock *rwlock); -__attribute__((visibility("default"))) void RWLockUnlock(struct rw_lock *rwlock); -__attribute__((visibility("default"))) void RWLockReadLock(struct rw_lock *rwlock); -__attribute__((visibility("default"))) void RWLockWriteLock(struct rw_lock *rwlock); -__attribute__((visibility("default"))) bool RWLockTryReadLock(struct rw_lock *rwlock); -__attribute__((visibility("default"))) bool RWLockTryWriteLock(struct rw_lock *rwlock); +__attribute__((visibility("default"))) void RWLockInit(struct RwLock *rwlock); +__attribute__((visibility("default"))) void RWLockDestroy(struct RwLock *rwlock); +__attribute__((visibility("default"))) ALWAYS_INLINE void ReadLock(struct RwLock *rwlock); +__attribute__((visibility("default"))) void RWLockUnlock(struct RwLock *rwlock); +__attribute__((visibility("default"))) void RWLockReadLock(struct RwLock *rwlock); +__attribute__((visibility("default"))) void RWLockWriteLock(struct RwLock *rwlock); +__attribute__((visibility("default"))) bool RWLockTryReadLock(struct RwLock *rwlock); +__attribute__((visibility("default"))) bool RWLockTryWriteLock(struct RwLock *rwlock); #ifndef MC_ON } // namespace panda::os::unix::memory::futex -- Gitee From 9d41780affd56be814e1c3ca29250e5dccd49933 Mon Sep 17 00:00:00 2001 From: Pavel Andrianov Date: Tue, 18 Apr 2023 16:05:25 +0300 Subject: [PATCH 11/17] One more missed rename Change-Id: Idbbe5d511ac634e37c8918e63cf4f146b3440eb7 --- platforms/unix/libpandabase/futex/mutex.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platforms/unix/libpandabase/futex/mutex.h b/platforms/unix/libpandabase/futex/mutex.h index e15798a01..824d13109 100644 --- a/platforms/unix/libpandabase/futex/mutex.h +++ b/platforms/unix/libpandabase/futex/mutex.h @@ -158,7 +158,7 @@ public: private: static_assert(std::atomic::is_always_lock_free); - struct rw_lock rwlock_; + struct RwLock rwlock_; NO_COPY_SEMANTIC(RWLock); NO_MOVE_SEMANTIC(RWLock); -- Gitee From 2276e9b4c3c679b49b6dc3590b1b31a8835adba7 Mon Sep 17 00:00:00 2001 From: Pavel Andrianov Date: Wed, 19 Apr 2023 09:27:27 +0300 Subject: [PATCH 12/17] Code style rename Change-Id: Id9fa26fb35c34fe9ee8e052f2eb3b3436666c152 --- platforms/unix/libpandabase/futex/fmutex.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platforms/unix/libpandabase/futex/fmutex.cpp b/platforms/unix/libpandabase/futex/fmutex.cpp index 8b116f65b..02673d6dc 100644 --- a/platforms/unix/libpandabase/futex/fmutex.cpp +++ b/platforms/unix/libpandabase/futex/fmutex.cpp @@ -165,8 +165,8 @@ static inline bool WaitBrieflyFor(ATOMIC_INT *addr, bool readlock) #ifndef MC_ON // We probably don't want to do syscall (switch context) when we use WaitBrieflyFor static constexpr uint32_t MAX_BACK_OFF = 10; - static constexpr uint32_t maxIter = 50; - for (uint32_t i = 1; i <= maxIter; i++) { + static constexpr uint32_t MAX_ITER = 50; + for (uint32_t i = 1; i <= MAX_ITER; i++) { BackOff(MIN(i, MAX_BACK_OFF)); #endif // Atomic with relaxed order reason: mutex synchronization -- Gitee From 1dff5d755a198fd97dce17f3e87ead66afae444b Mon Sep 17 00:00:00 2001 From: Pavel Andrianov Date: Thu, 20 Apr 2023 14:29:02 +0300 Subject: [PATCH 13/17] Revert exclusive_owner Change-Id: Iaf5bf7caa4d8f6f408c7e8f51b499af09e0451c6 --- platforms/unix/libpandabase/futex/fmutex.cpp | 23 +++++++++++++++++--- platforms/unix/libpandabase/futex/fmutex.h | 2 ++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/platforms/unix/libpandabase/futex/fmutex.cpp b/platforms/unix/libpandabase/futex/fmutex.cpp index 02673d6dc..3a704bd1c 100644 --- a/platforms/unix/libpandabase/futex/fmutex.cpp +++ b/platforms/unix/libpandabase/futex/fmutex.cpp @@ -565,7 +565,6 @@ void SignalCount(struct CondVar *const cond, int32_t to_wake) #endif } -// Empty string int *GetStateAddr(struct RwLock *const rwlock) { return reinterpret_cast(&rwlock->state); @@ -574,7 +573,13 @@ int *GetStateAddr(struct RwLock *const rwlock) bool HasExclusiveHolder(struct RwLock *const rwlock) { // Atomic with relaxed order reason: mutex synchronization - return ATOMIC_LOAD(&rwlock->state, memory_order_relaxed) == WRITE_LOCKED; + return ATOMIC_LOAD(&rwlock->exclusive_owner, memory_order_relaxed) != 0; +} + +bool IsExclusiveHeld(struct RwLock *const rwlock, THREAD_ID thread) +{ + // Atomic with relaxed order reason: mutex synchronization + return ATOMIC_LOAD(&rwlock->exclusive_owner, memory_order_relaxed) == thread; } void IncrementWaiters(struct RwLock *const rwlock) @@ -592,6 +597,7 @@ void RWLockInit(struct RwLock *const rwlock) { ATOMIC_STORE(&rwlock->state, 0, memory_order_relaxed); ATOMIC_STORE(&rwlock->waiters, 0, memory_order_relaxed); + ATOMIC_STORE(&rwlock->exclusive_owner, 0, memory_order_relaxed); } void RWLockDestroy(struct RwLock *const rwlock) @@ -603,6 +609,9 @@ void RWLockDestroy(struct RwLock *const rwlock) if (ATOMIC_LOAD(&rwlock->state, memory_order_relaxed) != 0) { FAIL_WITH_MESSAGE("RWLock destruction failed; state is non zero!"); // Atomic with relaxed order reason: mutex synchronization + } else if (ATOMIC_LOAD(&rwlock->exclusive_owner, memory_order_relaxed) != 0) { + FAIL_WITH_MESSAGE("RWLock destruction failed; RWLock has an owner!"); + // Atomic with relaxed order reason: mutex synchronization } else if (ATOMIC_LOAD(&rwlock->waiters, memory_order_relaxed) != 0) { FAIL_WITH_MESSAGE("RWLock destruction failed; RWLock has waiters!"); } @@ -702,6 +711,10 @@ void RWLockWriteLock(struct RwLock *const rwlock) // RWLock is held now // Atomic with relaxed order reason: mutex synchronization ASSERT(ATOMIC_LOAD(&rwlock->state, memory_order_relaxed) == WRITE_LOCKED); + // Atomic with relaxed order reason: mutex synchronization + ASSERT(ATOMIC_LOAD(&rwlock->exclusive_owner, memory_order_relaxed) == 0); + // Atomic with relaxed order reason: mutex synchronization + ATOMIC_STORE(&rwlock->exclusive_owner, current_tid, memory_order_relaxed); } bool RWLockTryReadLock(struct RwLock *const rwlock) @@ -751,6 +764,10 @@ bool RWLockTryWriteLock(struct RwLock *const rwlock) // RWLock is held now // Atomic with relaxed order reason: mutex synchronization ASSERT(ATOMIC_LOAD(&rwlock->state, memory_order_relaxed) == WRITE_LOCKED); + // Atomic with relaxed order reason: mutex synchronization + ASSERT(ATOMIC_LOAD(&rwlock->exclusive_owner, memory_order_relaxed) == 0); + // Atomic with relaxed order reason: mutex synchronization + ATOMIC_STORE(&rwlock->exclusive_owner, current_tid, memory_order_relaxed); return true; } @@ -792,7 +809,7 @@ void RWLockReadUnlock(struct RwLock *const rwlock) void RWLockWriteUnlock(struct RwLock *const rwlock) { ASSERT(current_tid != 0); - ASSERT(HasExclusiveHolder(rwlock)); + ASSERT(IsExclusiveHeld(rwlock, current_tid)); bool done = false; // Atomic with relaxed order reason: mutex synchronization diff --git a/platforms/unix/libpandabase/futex/fmutex.h b/platforms/unix/libpandabase/futex/fmutex.h index fc7319f48..0a167c91f 100644 --- a/platforms/unix/libpandabase/futex/fmutex.h +++ b/platforms/unix/libpandabase/futex/fmutex.h @@ -137,6 +137,8 @@ struct RwLock { ATOMIC(int32_t) state; // Number of waiters both for read and write locks. ATOMIC(int32_t) waiters; + // Exclusive owner. + alignas(alignof(uint32_t)) ATOMIC(THREAD_ID) exclusive_owner; #ifndef MC_ON std::array padding = {0}; #endif -- Gitee From 7c154b2acc1df585efac5887921be4aafc569268 Mon Sep 17 00:00:00 2001 From: Pavel Andrianov Date: Thu, 20 Apr 2023 17:05:33 +0300 Subject: [PATCH 14/17] Missed store Change-Id: Ia213ecf6bb754e94e97b673727f0a20ad31c34c5 --- platforms/unix/libpandabase/futex/fmutex.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/platforms/unix/libpandabase/futex/fmutex.cpp b/platforms/unix/libpandabase/futex/fmutex.cpp index 3a704bd1c..9c61b2766 100644 --- a/platforms/unix/libpandabase/futex/fmutex.cpp +++ b/platforms/unix/libpandabase/futex/fmutex.cpp @@ -817,6 +817,9 @@ void RWLockWriteUnlock(struct RwLock *const rwlock) // CAS is weak and might fail, do in loop while (!done) { if (LIKELY(cur_state == WRITE_LOCKED)) { + // Reset exclusive owner before changing state to avoid check failures if other thread sees UNLOCKED + // Atomic with relaxed order reason: mutex synchronization + ATOMIC_STORE(exclusive_owner_.store(0, std::memory_order_relaxed); // Change state to unlocked and do release store. // waiters load should not be reordered before state, so it's done with seq cst. // cur_state should be updated with fetched value on fail -- Gitee From 1459781c75d177682a58be79355f60ef0c07a5d1 Mon Sep 17 00:00:00 2001 From: Pavel Andrianov Date: Thu, 20 Apr 2023 20:24:23 +0300 Subject: [PATCH 15/17] Typo fix Change-Id: I907c2fb2e36af764ff79513de18516821895b3c7 --- platforms/unix/libpandabase/futex/fmutex.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platforms/unix/libpandabase/futex/fmutex.cpp b/platforms/unix/libpandabase/futex/fmutex.cpp index 9c61b2766..05fa7f542 100644 --- a/platforms/unix/libpandabase/futex/fmutex.cpp +++ b/platforms/unix/libpandabase/futex/fmutex.cpp @@ -819,7 +819,7 @@ void RWLockWriteUnlock(struct RwLock *const rwlock) if (LIKELY(cur_state == WRITE_LOCKED)) { // Reset exclusive owner before changing state to avoid check failures if other thread sees UNLOCKED // Atomic with relaxed order reason: mutex synchronization - ATOMIC_STORE(exclusive_owner_.store(0, std::memory_order_relaxed); + ATOMIC_STORE(&rwlock->exclusive_owner, 0, memory_order_relaxed); // Change state to unlocked and do release store. // waiters load should not be reordered before state, so it's done with seq cst. // cur_state should be updated with fetched value on fail -- Gitee From 3d6f650e8e20cc8b89b470aa8821f3e006064acc Mon Sep 17 00:00:00 2001 From: Pavel Andrianov Date: Fri, 21 Apr 2023 10:39:51 +0300 Subject: [PATCH 16/17] Move the dummies into h file Change-Id: Icbdcba29a669e2ccdccef88d58f02b2a37c95e59 --- platforms/unix/libpandabase/futex/mutex.cpp | 26 --------------------- platforms/unix/libpandabase/futex/mutex.h | 25 ++++++++++++++++---- 2 files changed, 20 insertions(+), 31 deletions(-) diff --git a/platforms/unix/libpandabase/futex/mutex.cpp b/platforms/unix/libpandabase/futex/mutex.cpp index 3f6216700..464d61acf 100644 --- a/platforms/unix/libpandabase/futex/mutex.cpp +++ b/platforms/unix/libpandabase/futex/mutex.cpp @@ -73,30 +73,4 @@ void Mutex::UnlockForOther(thread::ThreadId thread) { MutexUnlockForOther(&mutex_, thread); } - -// NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init) -RWLock::RWLock() -{ - RWLockInit(&rwlock_); -} - -RWLock::~RWLock() -{ - RWLockDestroy(&rwlock_); -} - -void RWLock::WriteLock() -{ - return RWLockWriteLock(&rwlock_); -} - -bool RWLock::TryReadLock() -{ - return RWLockTryReadLock(&rwlock_); -} - -bool RWLock::TryWriteLock() -{ - return RWLockTryWriteLock(&rwlock_); -} } // namespace panda::os::unix::memory::futex diff --git a/platforms/unix/libpandabase/futex/mutex.h b/platforms/unix/libpandabase/futex/mutex.h index 824d13109..9d7c714bb 100644 --- a/platforms/unix/libpandabase/futex/mutex.h +++ b/platforms/unix/libpandabase/futex/mutex.h @@ -135,9 +135,15 @@ private: class SHARED_CAPABILITY("mutex") RWLock { public: - PANDA_PUBLIC_API RWLock(); + PANDA_PUBLIC_API RWLock() + { + futex::RWLockInit(&rwlock_); + } - PANDA_PUBLIC_API ~RWLock(); + PANDA_PUBLIC_API ~RWLock() + { + futex::RWLockDestroy(&rwlock_); + } // ReadLock and ReadUnlock are used in mutator lock often, prefer inlining over call to libpandabase ALWAYS_INLINE void ReadLock() ACQUIRE_SHARED() @@ -150,11 +156,20 @@ public: futex::RWLockUnlock(&rwlock_); } - PANDA_PUBLIC_API void WriteLock() ACQUIRE(); + PANDA_PUBLIC_API void WriteLock() ACQUIRE() + { + return futex::RWLockWriteLock(&rwlock_); + } - PANDA_PUBLIC_API bool TryReadLock() TRY_ACQUIRE_SHARED(true); + PANDA_PUBLIC_API bool TryReadLock() TRY_ACQUIRE_SHARED(true) + { + return futex::RWLockTryReadLock(&rwlock_); + } - PANDA_PUBLIC_API bool TryWriteLock() TRY_ACQUIRE(true); + PANDA_PUBLIC_API bool TryWriteLock() TRY_ACQUIRE(true) + { + return futex::RWLockTryWriteLock(&rwlock_); + } private: static_assert(std::atomic::is_always_lock_free); -- Gitee From 19d493f37294821fa9798a50de739646e8670449 Mon Sep 17 00:00:00 2001 From: Pavel Andrianov Date: Mon, 24 Apr 2023 10:00:16 +0300 Subject: [PATCH 17/17] Disable warning Change-Id: Ifbf1a2b5b45384d95bf02051a3d845d6b8626983 --- platforms/unix/libpandabase/futex/fmutex.h | 1 + 1 file changed, 1 insertion(+) diff --git a/platforms/unix/libpandabase/futex/fmutex.h b/platforms/unix/libpandabase/futex/fmutex.h index 0a167c91f..0651230f3 100644 --- a/platforms/unix/libpandabase/futex/fmutex.h +++ b/platforms/unix/libpandabase/futex/fmutex.h @@ -132,6 +132,7 @@ static constexpr int32_t READ_INCREMENT = 1; // Extra padding to make RWLock 16 bytes long static constexpr size_t PADDING_SIZE = 1; +// NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init) struct RwLock { // -1 - write locked; 0 - unlocked; > 0 - read locked by state_ owners. ATOMIC(int32_t) state; -- Gitee