diff --git a/.circleci/config.yml b/.circleci/config.yml index 981cf9d..be49ddc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,12 +38,12 @@ jobs: command: | sudo sysctl vm.mmap_rnd_bits sudo sysctl vm.mmap_rnd_bits=28 - ./gtest --gtest_output=xml:gtest.xml + ./xenium_test --xenium_test_output=xml:xenium_test.xml working_directory: build - store_test_results: - path: build/gtest.xml + path: build/xenium_test.xml - store_artifacts: - path: build/gtest + path: build/xenium_test - when: condition: equal: [ << parameters.build-config >>, "Releass"] @@ -89,12 +89,12 @@ jobs: - run: name: "Run tests" command: | - ./gtest --gtest_output=xml:gtest.xml + ./xenium_test --xenium_test_output=xml:xenium_test.xml working_directory: build - store_test_results: - path: build/gtest.xml + path: build/xenium_test.xml - store_artifacts: - path: build/gtest + path: build/xenium_test - when: condition: equal: [ << parameters.build-config >>, "Releass"] @@ -138,12 +138,12 @@ jobs: working_directory: build - run: name: "Run tests" - command: .\gtest.exe --gtest_output=xml:gtest.xml + command: .\xenium_test.exe --xenium_test_output=xml:xenium_test.xml working_directory: build/<< parameters.build-config >> - store_test_results: - path: build/gtest.xml + path: build/xenium_test.xml - store_artifacts: - path: build/gtest.exe + path: build/xenium_test.exe workflows: diff --git a/3rdParty/config b/3rdParty/config deleted file mode 160000 index 46e0ab2..0000000 --- a/3rdParty/config +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 46e0ab2b6bc8228276d154cd339f9d8b8772d707 diff --git a/3rdParty/gtest b/3rdParty/gtest deleted file mode 160000 index 273f8cb..0000000 --- a/3rdParty/gtest +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 273f8cb059a4e7b089731036392422b5ef489791 diff --git a/3rdParty/gtest.cmake b/3rdParty/gtest.cmake deleted file mode 100644 index 1112ce6..0000000 --- a/3rdParty/gtest.cmake +++ /dev/null @@ -1,18 +0,0 @@ -set(GOOGLETEST_ROOT 3rdParty/gtest/googletest CACHE STRING "Google Test source root") - -include_directories(SYSTEM - ${PROJECT_SOURCE_DIR}/${GOOGLETEST_ROOT} - ${PROJECT_SOURCE_DIR}/${GOOGLETEST_ROOT}/include - ) - -set(GOOGLETEST_SOURCES - ${PROJECT_SOURCE_DIR}/${GOOGLETEST_ROOT}/src/gtest-all.cc - ${PROJECT_SOURCE_DIR}/${GOOGLETEST_ROOT}/src/gtest_main.cc - ) - -foreach(_source ${GOOGLETEST_SOURCES}) - set_source_files_properties(${_source} PROPERTIES GENERATED 1) -endforeach() - -add_library(googletest ${GOOGLETEST_SOURCES}) - diff --git a/3rdParty/json b/3rdParty/json deleted file mode 160000 index 879f27e..0000000 --- a/3rdParty/json +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 879f27eaf8b2d20f9187ca823256dc1da254fad5 diff --git a/3rdParty/libcds.cmake b/3rdParty/libcds.cmake deleted file mode 100644 index 5f50d4d..0000000 --- a/3rdParty/libcds.cmake +++ /dev/null @@ -1,27 +0,0 @@ - - -#include(ExternalProject) - -#ExternalProject_Add( -# libcds -# GIT_REPOSITORY https://github.com/khizmax/libcds.git -# GIT_TAG v2.3.3 - -# UPDATE_COMMAND "" -# CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=cds -#) - -#add_dependencies(benchmark libcds) - -#add_library(cds STATIC IMPORTED) - -#target_link_libraries(benchmark cds) - -#ExternalProject_Get_Property(libcds install_dir) -#message(${install_dir}) - -#set(cds_INCLUDE_DIRS "${CMAKE_SOURCE_DIR}/3rdparty/libcds/") -#set(cds_LIBRARIES "${CMAKE_SHARED_LIBRARY_PREFIX}cds${CMAKE_SHARED_LIBRARY_SUFFIX}") - -#TARGET_LINK_LIBRARIES(benchmark ${CDS_SHARED_LIBRARY} ) -#target_include_directories(benchmark PRIVATE ${cds_INCLUDE_DIRS}) diff --git a/CMakeLists.txt b/CMakeLists.txt index 96e7459..4700227 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,90 +1,384 @@ -cmake_minimum_required(VERSION 3.10) -project(xenium) +cmake_minimum_required(VERSION 3.30 FATAL_ERROR) -include(CheckCXXCompilerFlag) -include(CMakePushCheckState) +# +# Project +# +project( + xenium + VERSION 0.0.2 + LANGUAGES CXX + DESCRIPTION "Header-only library providing concurrent data structures and memory reclamation algorithms with customizable reclamation schemes" +) +# +# Target. +# +set(TARGET_NAME ${PROJECT_NAME}) + +# +# Settings +# set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(CMAKE_DEBUG_POSTFIX d) +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") -include(3rdParty/gtest.cmake) - -enable_testing() - -find_package(Threads REQUIRED) -find_package(Doxygen) - -option(WITH_TSAN "Build tests and benchmarks with ThreadSanitizer" ON) -option(BUILD_DOCUMENTATION "Create the HTML based documentation (requires Doxygen)" ${DOXYGEN_FOUND}) - -file(GLOB_RECURSE XENIUM_FILES xenium/*.hpp) +# +# Options +# +include(CheckCXXCompilerFlag) +include(CMakePushCheckState) +include(CMakeDependentOption) -file(GLOB_RECURSE TEST_FILES test/*.cpp) +option(XENIUM_CACHE_DEPS "Locally cache dependencies" ON) +option(XENIUM_BUILD_DOCS "Build documentation" OFF) +option(XENIUM_BUILD_TESTS "Build tests" ON) +option(XENIUM_BENCHMARK "Build Benchmark" ON) +option(XENIUM_WITH_TSAN "Build tests and benchmarks with ThreadSanitizer" ON) +# option(XENIUM_BENCHMARK_WITH_LIBDCDS "Build benchmarks with libcds" OFF) -file(GLOB_RECURSE BENCHMARK_FILES benchmarks/*.cpp) +# +# Runtime library +# +if(XENIUM_MSVC_RUNTIME_STATIC AND MSVC) + set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") +endif() -include_directories(.) +# +# Includes +# +include(GNUInstallDirs) +include(GenerateExportHeader) +include(CPM) +if(XENIUM_BUILD_DOCS) + find_package(Doxygen) +endif() -add_executable(gtest ${TEST_FILES} ${XENIUM_FILES}) -target_link_libraries(gtest googletest) -add_executable(benchmark ${BENCHMARK_FILES} ${XENIUM_FILES}) -target_include_directories( - benchmark - SYSTEM - PRIVATE - ./3rdParty/json/include - ./3rdParty/config/include -) +find_package(Threads REQUIRED) -if(CMAKE_THREAD_LIBS_INIT) - target_link_libraries(gtest "${CMAKE_THREAD_LIBS_INIT}") - target_link_libraries(benchmark "${CMAKE_THREAD_LIBS_INIT}") +# Cache the downloaded packets to avoid re-downloading them +if(XENIUM_CACHE_DEPS) + set(CPM_SOURCE_CACHE "${CMAKE_CURRENT_SOURCE_DIR}/cmake-build-deps/") endif() -if(WITH_LIBCDS) - find_package(LibCDS CONFIG REQUIRED) - target_link_libraries(benchmark LibCDS::cds) - target_compile_definitions(benchmark PRIVATE WITH_LIBCDS CDS_THREADING_CXX11) -endif() +if(XENIUM_BENCHMARK) +# No need to download taocpp_json separately, because taocpp_config already exposes it. +# https://github.com/taocpp/json/ -if(WITH_TSAN AND NOT MSVC) - cmake_push_check_state() - set(CMAKE_REQUIRED_FLAGS -fsanitize=thread) - check_cxx_compiler_flag("" TSAN_FLAG_WORKS) - cmake_pop_check_state() +# https://github.com/taocpp/config + CPMAddPackage( + NAME taocpp_config + GITHUB_REPOSITORY taocpp/config + GIT_TAG 93a455e103cdd8030e770dfff4e2d64d9057ed46 + EXCLUDE_FROM_ALL ON + SYSTEM ON + ) - if(TSAN_FLAG_WORKS) - string(APPEND CMAKE_CXX_FLAGS " -fsanitize=thread") - endif() + # if(XENIUM_BENCHMARK_WITH_LIBDCDS) +# + # CPMAddPackage( + # NAME libcds + # GITHUB_REPOSITORY khizmax/libcds + # GIT_TAG v2.3.3 + # EXCLUDE_FROM_ALL ON + # SYSTEM ON + # ) + # endif() endif() -if(MSVC) - target_compile_options(gtest PRIVATE /bigobj /W4) # /WX) - target_compile_options(benchmark PRIVATE /bigobj) # /W4 /WX) -else() - target_compile_options(gtest PRIVATE -Wall -Wextra -Werror -Wno-error=cpp) - target_compile_options(benchmark PRIVATE -Wall -Wextra -Werror -Wno-error=cpp) +if(XENIUM_BUILD_TESTS) + # https://github.com/google/googletest + CPMAddPackage( + NAME GoogleTest + GITHUB_REPOSITORY google/googletest + GIT_TAG v1.15.2 + EXCLUDE_FROM_ALL ON + SYSTEM ON + OPTIONS "INSTALL_GTEST OFF" "gtest_force_shared_crt ON" + ) endif() -if(CMAKE_BUILD_TYPE MATCHES Debug) - add_definitions(-DDEBUG) -elseif(CMAKE_BUILD_TYPE MATCHES RelWithDebug) - add_definitions(-DNDEBUG) -elseif(CMAKE_BUILD_TYPE MATCHES Release) - add_definitions(-DNDEBUG) -endif() +# +# Headers +# +set(XENIUM_HEADERS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include/${PROJECT_NAME}") +set(XENIUM_HEADERS +# cmake-format: sortable +"${XENIUM_HEADERS_DIR}/acquire_guard.hpp" +"${XENIUM_HEADERS_DIR}/aligned_object.hpp" +"${XENIUM_HEADERS_DIR}/backoff.hpp" +"${XENIUM_HEADERS_DIR}/chase_work_stealing_deque.hpp" +"${XENIUM_HEADERS_DIR}/detail" +"${XENIUM_HEADERS_DIR}/harris_michael_hash_map.hpp" +"${XENIUM_HEADERS_DIR}/harris_michael_list_based_set.hpp" +"${XENIUM_HEADERS_DIR}/hash.hpp" +"${XENIUM_HEADERS_DIR}/impl" +"${XENIUM_HEADERS_DIR}/kirsch_bounded_kfifo_queue.hpp" +"${XENIUM_HEADERS_DIR}/kirsch_kfifo_queue.hpp" +"${XENIUM_HEADERS_DIR}/left_right.hpp" +"${XENIUM_HEADERS_DIR}/marked_ptr.hpp" +"${XENIUM_HEADERS_DIR}/meta.hpp" +"${XENIUM_HEADERS_DIR}/michael_scott_queue.hpp" +"${XENIUM_HEADERS_DIR}/nikolaev_bounded_queue.hpp" +"${XENIUM_HEADERS_DIR}/nikolaev_queue.hpp" +"${XENIUM_HEADERS_DIR}/parameter.hpp" +"${XENIUM_HEADERS_DIR}/policy.hpp" +"${XENIUM_HEADERS_DIR}/ramalhete_queue.hpp" +"${XENIUM_HEADERS_DIR}/reclamation" +"${XENIUM_HEADERS_DIR}/seqlock.hpp" +"${XENIUM_HEADERS_DIR}/utils.hpp" +"${XENIUM_HEADERS_DIR}/vyukov_bounded_queue.hpp" +"${XENIUM_HEADERS_DIR}/vyukov_hash_map.hpp" +"${XENIUM_HEADERS_DIR}/acquire_guard.hpp" +"${XENIUM_HEADERS_DIR}/aligned_object.hpp" +"${XENIUM_HEADERS_DIR}/backoff.hpp" +"${XENIUM_HEADERS_DIR}/chase_work_stealing_deque.hpp" +"${XENIUM_HEADERS_DIR}/harris_michael_hash_map.hpp" +"${XENIUM_HEADERS_DIR}/harris_michael_list_based_set.hpp" +"${XENIUM_HEADERS_DIR}/hash.hpp" +"${XENIUM_HEADERS_DIR}/kirsch_bounded_kfifo_queue.hpp" +"${XENIUM_HEADERS_DIR}/kirsch_kfifo_queue.hpp" +"${XENIUM_HEADERS_DIR}/left_right.hpp" +"${XENIUM_HEADERS_DIR}/marked_ptr.hpp" +"${XENIUM_HEADERS_DIR}/meta.hpp" +"${XENIUM_HEADERS_DIR}/michael_scott_queue.hpp" +"${XENIUM_HEADERS_DIR}/nikolaev_bounded_queue.hpp" +"${XENIUM_HEADERS_DIR}/nikolaev_queue.hpp" +"${XENIUM_HEADERS_DIR}/parameter.hpp" +"${XENIUM_HEADERS_DIR}/policy.hpp" +"${XENIUM_HEADERS_DIR}/ramalhete_queue.hpp" +"${XENIUM_HEADERS_DIR}/seqlock.hpp" +"${XENIUM_HEADERS_DIR}/utils.hpp" +"${XENIUM_HEADERS_DIR}/vyukov_bounded_queue.hpp" +"${XENIUM_HEADERS_DIR}/vyukov_hash_map.hpp" +"${XENIUM_HEADERS_DIR}/detail/fixed_size_circular_array.hpp" +"${XENIUM_HEADERS_DIR}/detail/growing_circular_array.hpp" +"${XENIUM_HEADERS_DIR}/detail/hardware.hpp" +"${XENIUM_HEADERS_DIR}/detail/nikolaev_scq.hpp" +"${XENIUM_HEADERS_DIR}/detail/pointer_queue_traits.hpp" +"${XENIUM_HEADERS_DIR}/detail/port.hpp" +"${XENIUM_HEADERS_DIR}/detail/fixed_size_circular_array.hpp" +"${XENIUM_HEADERS_DIR}/detail/growing_circular_array.hpp" +"${XENIUM_HEADERS_DIR}/detail/hardware.hpp" +"${XENIUM_HEADERS_DIR}/detail/nikolaev_scq.hpp" +"${XENIUM_HEADERS_DIR}/detail/pointer_queue_traits.hpp" +"${XENIUM_HEADERS_DIR}/detail/port.hpp" +"${XENIUM_HEADERS_DIR}/impl/vyukov_hash_map.hpp" +"${XENIUM_HEADERS_DIR}/impl/vyukov_hash_map_traits.hpp" +"${XENIUM_HEADERS_DIR}/impl/vyukov_hash_map.hpp" +"${XENIUM_HEADERS_DIR}/impl/vyukov_hash_map_traits.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/detail" +"${XENIUM_HEADERS_DIR}/reclamation/generic_epoch_based.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/hazard_eras.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/hazard_pointer.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/impl" +"${XENIUM_HEADERS_DIR}/reclamation/lock_free_ref_count.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/quiescent_state_based.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/stamp_it.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/generic_epoch_based.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/hazard_eras.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/hazard_pointer.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/lock_free_ref_count.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/quiescent_state_based.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/stamp_it.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/detail/allocation_tracker.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/detail/concurrent_ptr.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/detail/deletable_object.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/detail/guard_ptr.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/detail/orphan.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/detail/perf_counter.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/detail/retire_list.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/detail/thread_block_list.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/detail/allocation_tracker.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/detail/concurrent_ptr.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/detail/deletable_object.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/detail/guard_ptr.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/detail/orphan.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/detail/perf_counter.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/detail/retire_list.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/detail/thread_block_list.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/impl/generic_epoch_based.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/impl/hazard_eras.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/impl/hazard_pointer.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/impl/lock_free_ref_count.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/impl/quiescent_state_based.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/impl/stamp_it.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/impl/generic_epoch_based.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/impl/hazard_eras.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/impl/hazard_pointer.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/impl/lock_free_ref_count.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/impl/quiescent_state_based.hpp" +"${XENIUM_HEADERS_DIR}/reclamation/impl/stamp_it.hpp" +) -if(BUILD_DOCUMENTATION) +set(XENIUM_LIB_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include") +set(XENIUM_INCLUDE_DIRS_PUB "${XENIUM_LIB_INCLUDE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/include") + +if(XENIUM_BUILD_DOCS) if(NOT DOXYGEN_FOUND) message(FATAL_ERROR "Doxygen is needed to build the documentation.") endif() add_custom_target(doc - COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/doc/Doxyfile - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/doc + COMMAND ${DOXYGEN_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/doc/Doxyfile" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/doc" COMMENT "Generating API documentation with Doxygen" VERBATIM) endif() -add_test(AllTests gtest) +add_library(${TARGET_NAME} INTERFACE ${XENIUM_HEADERS}) +add_library(${TARGET_NAME}::${TARGET_NAME} ALIAS ${TARGET_NAME}) +# expose include dir +target_include_directories(${TARGET_NAME} INTERFACE ${XENIUM_INCLUDE_DIRS_PUB}) + +# test +if(XENIUM_BUILD_TESTS) + include(GoogleTest) + set(TEST_NAME ${TARGET_NAME}_test) + + set(XENIUM_TESTS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/test") + + set(XENIUM_TESTS_HEADERS + "${XENIUM_TESTS_DIR}/helpers.hpp" + ) + + set(XENIUM_TESTS_SRC + "${XENIUM_TESTS_DIR}/chase_work_stealing_deque_test.cpp" + "${XENIUM_TESTS_DIR}/concurrent_ptr_test.cpp" + "${XENIUM_TESTS_DIR}/harris_michael_hash_map_test.cpp" + "${XENIUM_TESTS_DIR}/harris_michael_list_based_set_test.cpp" + "${XENIUM_TESTS_DIR}/kirsch_bounded_kfifo_queue_test.cpp" + "${XENIUM_TESTS_DIR}/kirsch_kfifo_queue_test.cpp" + "${XENIUM_TESTS_DIR}/left_right_test.cpp" + "${XENIUM_TESTS_DIR}/main.cpp" + "${XENIUM_TESTS_DIR}/marked_ptr_test.cpp" + "${XENIUM_TESTS_DIR}/michael_scott_queue_test.cpp" + "${XENIUM_TESTS_DIR}/nikolaev_bounded_queue_test.cpp" + "${XENIUM_TESTS_DIR}/nikolaev_queue_test.cpp" + "${XENIUM_TESTS_DIR}/parameter_test.cpp" + "${XENIUM_TESTS_DIR}/ramalhete_queue_test.cpp" + "${XENIUM_TESTS_DIR}/sanitize_test.cpp" + "${XENIUM_TESTS_DIR}/seqlock_test.cpp" + "${XENIUM_TESTS_DIR}/vyukov_bounded_queue_test.cpp" + "${XENIUM_TESTS_DIR}/vyukov_hash_map_test.cpp" + "${XENIUM_TESTS_DIR}/chase_work_stealing_deque_test.cpp" + "${XENIUM_TESTS_DIR}/concurrent_ptr_test.cpp" + "${XENIUM_TESTS_DIR}/harris_michael_hash_map_test.cpp" + "${XENIUM_TESTS_DIR}/harris_michael_list_based_set_test.cpp" + "${XENIUM_TESTS_DIR}/kirsch_bounded_kfifo_queue_test.cpp" + "${XENIUM_TESTS_DIR}/kirsch_kfifo_queue_test.cpp" + "${XENIUM_TESTS_DIR}/left_right_test.cpp" + "${XENIUM_TESTS_DIR}/main.cpp" + "${XENIUM_TESTS_DIR}/marked_ptr_test.cpp" + "${XENIUM_TESTS_DIR}/michael_scott_queue_test.cpp" + "${XENIUM_TESTS_DIR}/nikolaev_bounded_queue_test.cpp" + "${XENIUM_TESTS_DIR}/nikolaev_queue_test.cpp" + "${XENIUM_TESTS_DIR}/parameter_test.cpp" + "${XENIUM_TESTS_DIR}/ramalhete_queue_test.cpp" + "${XENIUM_TESTS_DIR}/sanitize_test.cpp" + "${XENIUM_TESTS_DIR}/seqlock_test.cpp" + "${XENIUM_TESTS_DIR}/vyukov_bounded_queue_test.cpp" + "${XENIUM_TESTS_DIR}/vyukov_hash_map_test.cpp" + "${XENIUM_TESTS_DIR}/detail/nikolaev_scq_test.cpp" + "${XENIUM_TESTS_DIR}/detail/nikolaev_scq_test.cpp" + "${XENIUM_TESTS_DIR}/reclamation/generic_epoch_based_test.cpp" + "${XENIUM_TESTS_DIR}/reclamation/hazard_eras_test.cpp" + "${XENIUM_TESTS_DIR}/reclamation/hazard_pointer_test.cpp" + "${XENIUM_TESTS_DIR}/reclamation/lock_free_ref_count_test.cpp" + "${XENIUM_TESTS_DIR}/reclamation/quiescent_state_based_test.cpp" + "${XENIUM_TESTS_DIR}/reclamation/stamp_it_test.cpp" + "${XENIUM_TESTS_DIR}/reclamation/generic_epoch_based_test.cpp" + "${XENIUM_TESTS_DIR}/reclamation/hazard_eras_test.cpp" + "${XENIUM_TESTS_DIR}/reclamation/hazard_pointer_test.cpp" + "${XENIUM_TESTS_DIR}/reclamation/lock_free_ref_count_test.cpp" + "${XENIUM_TESTS_DIR}/reclamation/quiescent_state_based_test.cpp" + "${XENIUM_TESTS_DIR}/reclamation/stamp_it_test.cpp" + ) + add_executable(${TEST_NAME} ${XENIUM_TESTS_HEADERS} ${XENIUM_TESTS_SRC}) + target_link_libraries( + ${TEST_NAME} + PRIVATE + GTest::gtest_main + ${TARGET_NAME} + ) + + # + # Target compile option + # + if(MSVC) + target_compile_options(${TEST_NAME} PRIVATE /bigobj /W4) + else() + target_compile_options(${TEST_NAME} PRIVATE -Wall -Wextra -Werror -Wno-error=cpp) + endif() + +endif() + +if(XENIUM_BENCHMARK) + + set(XENIUM_BENCHMARK_DIR "${CMAKE_CURRENT_SOURCE_DIR}/benchmarks") + set(XENIUM_BENCHMARK_HEADERS + "${XENIUM_BENCHMARK_DIR}/benchmark.hpp" + "${XENIUM_BENCHMARK_DIR}/config.hpp" + "${XENIUM_BENCHMARK_DIR}/descriptor.hpp" + "${XENIUM_BENCHMARK_DIR}/execution.hpp" + "${XENIUM_BENCHMARK_DIR}/hash_maps.hpp" + "${XENIUM_BENCHMARK_DIR}/queues.hpp" + "${XENIUM_BENCHMARK_DIR}/reclaimers.hpp" + "${XENIUM_BENCHMARK_DIR}/report.hpp" + "${XENIUM_BENCHMARK_DIR}/workload.hpp" + ) + set(XENIUM_BENCHMARK_SRC + "${XENIUM_BENCHMARK_DIR}/benchmark.cpp" + "${XENIUM_BENCHMARK_DIR}/execution.cpp" + "${XENIUM_BENCHMARK_DIR}/hash_map_benchmark.cpp" + "${XENIUM_BENCHMARK_DIR}/main.cpp" + "${XENIUM_BENCHMARK_DIR}/queue_benchmark.cpp" + "${XENIUM_BENCHMARK_DIR}/report.cpp" + "${XENIUM_BENCHMARK_DIR}/workload.cpp" + "${XENIUM_BENCHMARK_DIR}/benchmark.cpp" + "${XENIUM_BENCHMARK_DIR}/execution.cpp" + "${XENIUM_BENCHMARK_DIR}/hash_map_benchmark.cpp" + "${XENIUM_BENCHMARK_DIR}/main.cpp" + "${XENIUM_BENCHMARK_DIR}/queue_benchmark.cpp" + "${XENIUM_BENCHMARK_DIR}/report.cpp" + "${XENIUM_BENCHMARK_DIR}/workload.cpp" + ) + + add_executable(benchmark ${XENIUM_BENCHMARK_HEADERS} ${XENIUM_BENCHMARK_SRC}) + + target_link_libraries( + benchmark + PRIVATE + ${TARGET_NAME} + taocpp-json + taocpp-config + # $<$:cds> + ) + + if(MSVC) + target_compile_options(benchmark PRIVATE /bigobj /W4 ) + else() + target_compile_options(benchmark PRIVATE /bigobj /W4) + endif() + + # else() + # if(XENIUM_BENCHMARK_WITH_LIBDCDS) + # target_compile_options(benchmark PRIVATE -Wall -Wextra -Werror -Wno-error=cpp WITH_LIBCDS) + # else() + # target_compile_options(benchmark PRIVATE -Wall -Wextra -Werror -Wno-error=cpp) + # endif() +endif() + +if(XENIUM_WITH_TSAN AND NOT MSVC) + cmake_push_check_state() + set(CMAKE_REQUIRED_FLAGS -fsanitize=thread) + check_cxx_compiler_flag("" TSAN_FLAG_WORKS) + cmake_pop_check_state() + + if(TSAN_FLAG_WORKS) + string(APPEND CMAKE_CXX_FLAGS " -fsanitize=thread") + endif() +endif() \ No newline at end of file diff --git a/benchmarks/main.cpp b/benchmarks/main.cpp index a6e9d3e..c4f6a29 100644 --- a/benchmarks/main.cpp +++ b/benchmarks/main.cpp @@ -8,12 +8,8 @@ #define __int128_t std::int64_t #endif -#include - -#include -#include - -#include +#include +#include #include "execution.hpp" @@ -157,17 +153,16 @@ class runner { }; runner::runner(const options& opts) : _reportfile(opts.report) { - tao::config::internal::configurator configurator; - configurator.parse(tao::config::pegtl::file_input(opts.configfile)); + // carica la configurazione di base + _config = tao::config::from_file(opts.configfile); + // applica override dai parametri della riga di comando for (const auto& param : opts.params) { - // TODO - error handling std::cout << "param: " << param << std::endl; - configurator.parse(tao::config::pegtl_input_t(param, "command line param")); + auto kv = split_key_value(param); + _config[kv.key] = tao::config::from_string(kv.value, "param error"); } - _config = configurator.process(tao::config::schema::builtin()); - load_config(); } diff --git a/cmake/CPM.cmake b/cmake/CPM.cmake new file mode 100644 index 0000000..eea2921 --- /dev/null +++ b/cmake/CPM.cmake @@ -0,0 +1,1291 @@ +# CPM.cmake - CMake's missing package manager +# =========================================== +# See https://github.com/cpm-cmake/CPM.cmake for usage and update instructions. +# +# MIT License +# ----------- +#[[ + Copyright (c) 2019-2023 Lars Melchior and contributors + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +]] + +cmake_minimum_required(VERSION 3.14 FATAL_ERROR) + +# Initialize logging prefix +if(NOT CPM_INDENT) + set(CPM_INDENT + "CPM:" + CACHE INTERNAL "" + ) +endif() + +if(NOT COMMAND cpm_message) + function(cpm_message) + message(${ARGV}) + endfunction() +endif() + +if(DEFINED EXTRACTED_CPM_VERSION) + set(CURRENT_CPM_VERSION "${EXTRACTED_CPM_VERSION}${CPM_DEVELOPMENT}") +else() + set(CURRENT_CPM_VERSION 0.40.8) +endif() + +get_filename_component(CPM_CURRENT_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}" REALPATH) +if(CPM_DIRECTORY) + if(NOT CPM_DIRECTORY STREQUAL CPM_CURRENT_DIRECTORY) + if(CPM_VERSION VERSION_LESS CURRENT_CPM_VERSION) + message( + AUTHOR_WARNING + "${CPM_INDENT} \ +A dependency is using a more recent CPM version (${CURRENT_CPM_VERSION}) than the current project (${CPM_VERSION}). \ +It is recommended to upgrade CPM to the most recent version. \ +See https://github.com/cpm-cmake/CPM.cmake for more information." + ) + endif() + if(${CMAKE_VERSION} VERSION_LESS "3.17.0") + include(FetchContent) + endif() + return() + endif() + + get_property( + CPM_INITIALIZED GLOBAL "" + PROPERTY CPM_INITIALIZED + SET + ) + if(CPM_INITIALIZED) + return() + endif() +endif() + +if(CURRENT_CPM_VERSION MATCHES "development-version") + message( + WARNING "${CPM_INDENT} Your project is using an unstable development version of CPM.cmake. \ +Please update to a recent release if possible. \ +See https://github.com/cpm-cmake/CPM.cmake for details." + ) +endif() + +set_property(GLOBAL PROPERTY CPM_INITIALIZED true) + +macro(cpm_set_policies) + # the policy allows us to change options without caching + cmake_policy(SET CMP0077 NEW) + set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) + + # the policy allows us to change set(CACHE) without caching + if(POLICY CMP0126) + cmake_policy(SET CMP0126 NEW) + set(CMAKE_POLICY_DEFAULT_CMP0126 NEW) + endif() + + # The policy uses the download time for timestamp, instead of the timestamp in the archive. This + # allows for proper rebuilds when a projects url changes + if(POLICY CMP0135) + cmake_policy(SET CMP0135 NEW) + set(CMAKE_POLICY_DEFAULT_CMP0135 NEW) + endif() + + # treat relative git repository paths as being relative to the parent project's remote + if(POLICY CMP0150) + cmake_policy(SET CMP0150 NEW) + set(CMAKE_POLICY_DEFAULT_CMP0150 NEW) + endif() +endmacro() +cpm_set_policies() + +option(CPM_USE_LOCAL_PACKAGES "Always try to use `find_package` to get dependencies" + $ENV{CPM_USE_LOCAL_PACKAGES} +) +option(CPM_LOCAL_PACKAGES_ONLY "Only use `find_package` to get dependencies" + $ENV{CPM_LOCAL_PACKAGES_ONLY} +) +option(CPM_DOWNLOAD_ALL "Always download dependencies from source" $ENV{CPM_DOWNLOAD_ALL}) +option(CPM_DONT_UPDATE_MODULE_PATH "Don't update the module path to allow using find_package" + $ENV{CPM_DONT_UPDATE_MODULE_PATH} +) +option(CPM_DONT_CREATE_PACKAGE_LOCK "Don't create a package lock file in the binary path" + $ENV{CPM_DONT_CREATE_PACKAGE_LOCK} +) +option(CPM_INCLUDE_ALL_IN_PACKAGE_LOCK + "Add all packages added through CPM.cmake to the package lock" + $ENV{CPM_INCLUDE_ALL_IN_PACKAGE_LOCK} +) +option(CPM_USE_NAMED_CACHE_DIRECTORIES + "Use additional directory of package name in cache on the most nested level." + $ENV{CPM_USE_NAMED_CACHE_DIRECTORIES} +) + +set(CPM_VERSION + ${CURRENT_CPM_VERSION} + CACHE INTERNAL "" +) +set(CPM_DIRECTORY + ${CPM_CURRENT_DIRECTORY} + CACHE INTERNAL "" +) +set(CPM_FILE + ${CMAKE_CURRENT_LIST_FILE} + CACHE INTERNAL "" +) +set(CPM_PACKAGES + "" + CACHE INTERNAL "" +) +set(CPM_DRY_RUN + OFF + CACHE INTERNAL "Don't download or configure dependencies (for testing)" +) + +if(DEFINED ENV{CPM_SOURCE_CACHE}) + set(CPM_SOURCE_CACHE_DEFAULT $ENV{CPM_SOURCE_CACHE}) +else() + set(CPM_SOURCE_CACHE_DEFAULT OFF) +endif() + +set(CPM_SOURCE_CACHE + ${CPM_SOURCE_CACHE_DEFAULT} + CACHE PATH "Directory to download CPM dependencies" +) + +if(NOT CPM_DONT_UPDATE_MODULE_PATH AND NOT DEFINED CMAKE_FIND_PACKAGE_REDIRECTS_DIR) + set(CPM_MODULE_PATH + "${CMAKE_BINARY_DIR}/CPM_modules" + CACHE INTERNAL "" + ) + # remove old modules + file(REMOVE_RECURSE ${CPM_MODULE_PATH}) + file(MAKE_DIRECTORY ${CPM_MODULE_PATH}) + # locally added CPM modules should override global packages + set(CMAKE_MODULE_PATH "${CPM_MODULE_PATH};${CMAKE_MODULE_PATH}") +endif() + +if(NOT CPM_DONT_CREATE_PACKAGE_LOCK) + set(CPM_PACKAGE_LOCK_FILE + "${CMAKE_BINARY_DIR}/cpm-package-lock.cmake" + CACHE INTERNAL "" + ) + file(WRITE ${CPM_PACKAGE_LOCK_FILE} + "# CPM Package Lock\n# This file should be committed to version control\n\n" + ) +endif() + +include(FetchContent) + +# Try to infer package name from git repository uri (path or url) +function(cpm_package_name_from_git_uri URI RESULT) + if("${URI}" MATCHES "([^/:]+)/?.git/?$") + set(${RESULT} + ${CMAKE_MATCH_1} + PARENT_SCOPE + ) + else() + unset(${RESULT} PARENT_SCOPE) + endif() +endfunction() + +# Try to infer package name and version from a url +function(cpm_package_name_and_ver_from_url url outName outVer) + if(url MATCHES "[/\\?]([a-zA-Z0-9_\\.-]+)\\.(tar|tar\\.gz|tar\\.bz2|zip|ZIP)(\\?|/|$)") + # We matched an archive + set(filename "${CMAKE_MATCH_1}") + + if(filename MATCHES "([a-zA-Z0-9_\\.-]+)[_-]v?(([0-9]+\\.)*[0-9]+[a-zA-Z0-9]*)") + # We matched - (ie foo-1.2.3) + set(${outName} + "${CMAKE_MATCH_1}" + PARENT_SCOPE + ) + set(${outVer} + "${CMAKE_MATCH_2}" + PARENT_SCOPE + ) + elseif(filename MATCHES "(([0-9]+\\.)+[0-9]+[a-zA-Z0-9]*)") + # We couldn't find a name, but we found a version + # + # In many cases (which we don't handle here) the url would look something like + # `irrelevant/ACTUAL_PACKAGE_NAME/irrelevant/1.2.3.zip`. In such a case we can't possibly + # distinguish the package name from the irrelevant bits. Moreover if we try to match the + # package name from the filename, we'd get bogus at best. + unset(${outName} PARENT_SCOPE) + set(${outVer} + "${CMAKE_MATCH_1}" + PARENT_SCOPE + ) + else() + # Boldly assume that the file name is the package name. + # + # Yes, something like `irrelevant/ACTUAL_NAME/irrelevant/download.zip` will ruin our day, but + # such cases should be quite rare. No popular service does this... we think. + set(${outName} + "${filename}" + PARENT_SCOPE + ) + unset(${outVer} PARENT_SCOPE) + endif() + else() + # No ideas yet what to do with non-archives + unset(${outName} PARENT_SCOPE) + unset(${outVer} PARENT_SCOPE) + endif() +endfunction() + +function(cpm_find_package NAME VERSION) + string(REPLACE " " ";" EXTRA_ARGS "${ARGN}") + find_package(${NAME} ${VERSION} ${EXTRA_ARGS} QUIET) + if(${CPM_ARGS_NAME}_FOUND) + if(DEFINED ${CPM_ARGS_NAME}_VERSION) + set(VERSION ${${CPM_ARGS_NAME}_VERSION}) + endif() + cpm_message(STATUS "${CPM_INDENT} Using local package ${CPM_ARGS_NAME}@${VERSION}") + CPMRegisterPackage(${CPM_ARGS_NAME} "${VERSION}") + set(CPM_PACKAGE_FOUND + YES + PARENT_SCOPE + ) + else() + set(CPM_PACKAGE_FOUND + NO + PARENT_SCOPE + ) + endif() +endfunction() + +# Create a custom FindXXX.cmake module for a CPM package This prevents `find_package(NAME)` from +# finding the system library +function(cpm_create_module_file Name) + if(NOT CPM_DONT_UPDATE_MODULE_PATH) + if(DEFINED CMAKE_FIND_PACKAGE_REDIRECTS_DIR) + # Redirect find_package calls to the CPM package. This is what FetchContent does when you set + # OVERRIDE_FIND_PACKAGE. The CMAKE_FIND_PACKAGE_REDIRECTS_DIR works for find_package in CONFIG + # mode, unlike the Find${Name}.cmake fallback. CMAKE_FIND_PACKAGE_REDIRECTS_DIR is not defined + # in script mode, or in CMake < 3.24. + # https://cmake.org/cmake/help/latest/module/FetchContent.html#fetchcontent-find-package-integration-examples + string(TOLOWER ${Name} NameLower) + file(WRITE ${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}/${NameLower}-config.cmake + "include(\"\${CMAKE_CURRENT_LIST_DIR}/${NameLower}-extra.cmake\" OPTIONAL)\n" + "include(\"\${CMAKE_CURRENT_LIST_DIR}/${Name}Extra.cmake\" OPTIONAL)\n" + ) + file(WRITE ${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}/${NameLower}-config-version.cmake + "set(PACKAGE_VERSION_COMPATIBLE TRUE)\n" "set(PACKAGE_VERSION_EXACT TRUE)\n" + ) + else() + file(WRITE ${CPM_MODULE_PATH}/Find${Name}.cmake + "include(\"${CPM_FILE}\")\n${ARGN}\nset(${Name}_FOUND TRUE)" + ) + endif() + endif() +endfunction() + +# Find a package locally or fallback to CPMAddPackage +function(CPMFindPackage) + set(oneValueArgs NAME VERSION GIT_TAG FIND_PACKAGE_ARGUMENTS) + + cmake_parse_arguments(CPM_ARGS "" "${oneValueArgs}" "" ${ARGN}) + + if(NOT DEFINED CPM_ARGS_VERSION) + if(DEFINED CPM_ARGS_GIT_TAG) + cpm_get_version_from_git_tag("${CPM_ARGS_GIT_TAG}" CPM_ARGS_VERSION) + endif() + endif() + + set(downloadPackage ${CPM_DOWNLOAD_ALL}) + if(DEFINED CPM_DOWNLOAD_${CPM_ARGS_NAME}) + set(downloadPackage ${CPM_DOWNLOAD_${CPM_ARGS_NAME}}) + elseif(DEFINED ENV{CPM_DOWNLOAD_${CPM_ARGS_NAME}}) + set(downloadPackage $ENV{CPM_DOWNLOAD_${CPM_ARGS_NAME}}) + endif() + if(downloadPackage) + CPMAddPackage(${ARGN}) + cpm_export_variables(${CPM_ARGS_NAME}) + return() + endif() + + cpm_find_package(${CPM_ARGS_NAME} "${CPM_ARGS_VERSION}" ${CPM_ARGS_FIND_PACKAGE_ARGUMENTS}) + + if(NOT CPM_PACKAGE_FOUND) + CPMAddPackage(${ARGN}) + cpm_export_variables(${CPM_ARGS_NAME}) + endif() + +endfunction() + +# checks if a package has been added before +function(cpm_check_if_package_already_added CPM_ARGS_NAME CPM_ARGS_VERSION) + if("${CPM_ARGS_NAME}" IN_LIST CPM_PACKAGES) + CPMGetPackageVersion(${CPM_ARGS_NAME} CPM_PACKAGE_VERSION) + if("${CPM_PACKAGE_VERSION}" VERSION_LESS "${CPM_ARGS_VERSION}") + message( + WARNING + "${CPM_INDENT} Requires a newer version of ${CPM_ARGS_NAME} (${CPM_ARGS_VERSION}) than currently included (${CPM_PACKAGE_VERSION})." + ) + endif() + cpm_get_fetch_properties(${CPM_ARGS_NAME}) + set(${CPM_ARGS_NAME}_ADDED NO) + set(CPM_PACKAGE_ALREADY_ADDED + YES + PARENT_SCOPE + ) + cpm_export_variables(${CPM_ARGS_NAME}) + else() + set(CPM_PACKAGE_ALREADY_ADDED + NO + PARENT_SCOPE + ) + endif() +endfunction() + +# Parse the argument of CPMAddPackage in case a single one was provided and convert it to a list of +# arguments which can then be parsed idiomatically. For example gh:foo/bar@1.2.3 will be converted +# to: GITHUB_REPOSITORY;foo/bar;VERSION;1.2.3 +function(cpm_parse_add_package_single_arg arg outArgs) + # Look for a scheme + if("${arg}" MATCHES "^([a-zA-Z]+):(.+)$") + string(TOLOWER "${CMAKE_MATCH_1}" scheme) + set(uri "${CMAKE_MATCH_2}") + + # Check for CPM-specific schemes + if(scheme STREQUAL "gh") + set(out "GITHUB_REPOSITORY;${uri}") + set(packageType "git") + elseif(scheme STREQUAL "gl") + set(out "GITLAB_REPOSITORY;${uri}") + set(packageType "git") + elseif(scheme STREQUAL "bb") + set(out "BITBUCKET_REPOSITORY;${uri}") + set(packageType "git") + # A CPM-specific scheme was not found. Looks like this is a generic URL so try to determine + # type + elseif(arg MATCHES ".git/?(@|#|$)") + set(out "GIT_REPOSITORY;${arg}") + set(packageType "git") + else() + # Fall back to a URL + set(out "URL;${arg}") + set(packageType "archive") + + # We could also check for SVN since FetchContent supports it, but SVN is so rare these days. + # We just won't bother with the additional complexity it will induce in this function. SVN is + # done by multi-arg + endif() + else() + if(arg MATCHES ".git/?(@|#|$)") + set(out "GIT_REPOSITORY;${arg}") + set(packageType "git") + else() + # Give up + message(FATAL_ERROR "${CPM_INDENT} Can't determine package type of '${arg}'") + endif() + endif() + + # For all packages we interpret @... as version. Only replace the last occurrence. Thus URIs + # containing '@' can be used + string(REGEX REPLACE "@([^@]+)$" ";VERSION;\\1" out "${out}") + + # Parse the rest according to package type + if(packageType STREQUAL "git") + # For git repos we interpret #... as a tag or branch or commit hash + string(REGEX REPLACE "#([^#]+)$" ";GIT_TAG;\\1" out "${out}") + elseif(packageType STREQUAL "archive") + # For archives we interpret #... as a URL hash. + string(REGEX REPLACE "#([^#]+)$" ";URL_HASH;\\1" out "${out}") + # We don't try to parse the version if it's not provided explicitly. cpm_get_version_from_url + # should do this at a later point + else() + # We should never get here. This is an assertion and hitting it means there's a problem with the + # code above. A packageType was set, but not handled by this if-else. + message(FATAL_ERROR "${CPM_INDENT} Unsupported package type '${packageType}' of '${arg}'") + endif() + + set(${outArgs} + ${out} + PARENT_SCOPE + ) +endfunction() + +# Check that the working directory for a git repo is clean +function(cpm_check_git_working_dir_is_clean repoPath gitTag isClean) + + find_package(Git REQUIRED) + + if(NOT GIT_EXECUTABLE) + # No git executable, assume directory is clean + set(${isClean} + TRUE + PARENT_SCOPE + ) + return() + endif() + + # check for uncommitted changes + execute_process( + COMMAND ${GIT_EXECUTABLE} status --porcelain + RESULT_VARIABLE resultGitStatus + OUTPUT_VARIABLE repoStatus + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET + WORKING_DIRECTORY ${repoPath} + ) + if(resultGitStatus) + # not supposed to happen, assume clean anyway + message(WARNING "${CPM_INDENT} Calling git status on folder ${repoPath} failed") + set(${isClean} + TRUE + PARENT_SCOPE + ) + return() + endif() + + if(NOT "${repoStatus}" STREQUAL "") + set(${isClean} + FALSE + PARENT_SCOPE + ) + return() + endif() + + # check for committed changes + execute_process( + COMMAND ${GIT_EXECUTABLE} diff -s --exit-code ${gitTag} + RESULT_VARIABLE resultGitDiff + OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_QUIET + WORKING_DIRECTORY ${repoPath} + ) + + if(${resultGitDiff} EQUAL 0) + set(${isClean} + TRUE + PARENT_SCOPE + ) + else() + set(${isClean} + FALSE + PARENT_SCOPE + ) + endif() + +endfunction() + +# Add PATCH_COMMAND to CPM_ARGS_UNPARSED_ARGUMENTS. This method consumes a list of files in ARGN +# then generates a `PATCH_COMMAND` appropriate for `ExternalProject_Add()`. This command is appended +# to the parent scope's `CPM_ARGS_UNPARSED_ARGUMENTS`. +function(cpm_add_patches) + # Return if no patch files are supplied. + if(NOT ARGN) + return() + endif() + + # Find the patch program. + find_program(PATCH_EXECUTABLE patch) + if(CMAKE_HOST_WIN32 AND NOT PATCH_EXECUTABLE) + # The Windows git executable is distributed with patch.exe. Find the path to the executable, if + # it exists, then search `../usr/bin` and `../../usr/bin` for patch.exe. + find_package(Git QUIET) + if(GIT_EXECUTABLE) + get_filename_component(extra_search_path ${GIT_EXECUTABLE} DIRECTORY) + get_filename_component(extra_search_path_1up ${extra_search_path} DIRECTORY) + get_filename_component(extra_search_path_2up ${extra_search_path_1up} DIRECTORY) + find_program( + PATCH_EXECUTABLE patch HINTS "${extra_search_path_1up}/usr/bin" + "${extra_search_path_2up}/usr/bin" + ) + endif() + endif() + if(NOT PATCH_EXECUTABLE) + message(FATAL_ERROR "Couldn't find `patch` executable to use with PATCHES keyword.") + endif() + + # Create a temporary + set(temp_list ${CPM_ARGS_UNPARSED_ARGUMENTS}) + + # Ensure each file exists (or error out) and add it to the list. + set(first_item True) + foreach(PATCH_FILE ${ARGN}) + # Make sure the patch file exists, if we can't find it, try again in the current directory. + if(NOT EXISTS "${PATCH_FILE}") + if(NOT EXISTS "${CMAKE_CURRENT_LIST_DIR}/${PATCH_FILE}") + message(FATAL_ERROR "Couldn't find patch file: '${PATCH_FILE}'") + endif() + set(PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/${PATCH_FILE}") + endif() + + # Convert to absolute path for use with patch file command. + get_filename_component(PATCH_FILE "${PATCH_FILE}" ABSOLUTE) + + # The first patch entry must be preceded by "PATCH_COMMAND" while the following items are + # preceded by "&&". + if(first_item) + set(first_item False) + list(APPEND temp_list "PATCH_COMMAND") + else() + list(APPEND temp_list "&&") + endif() + # Add the patch command to the list + list(APPEND temp_list "${PATCH_EXECUTABLE}" "-p1" "<" "${PATCH_FILE}") + endforeach() + + # Move temp out into parent scope. + set(CPM_ARGS_UNPARSED_ARGUMENTS + ${temp_list} + PARENT_SCOPE + ) + +endfunction() + +# method to overwrite internal FetchContent properties, to allow using CPM.cmake to overload +# FetchContent calls. As these are internal cmake properties, this method should be used carefully +# and may need modification in future CMake versions. Source: +# https://github.com/Kitware/CMake/blob/dc3d0b5a0a7d26d43d6cfeb511e224533b5d188f/Modules/FetchContent.cmake#L1152 +function(cpm_override_fetchcontent contentName) + cmake_parse_arguments(PARSE_ARGV 1 arg "" "SOURCE_DIR;BINARY_DIR" "") + if(NOT "${arg_UNPARSED_ARGUMENTS}" STREQUAL "") + message(FATAL_ERROR "${CPM_INDENT} Unsupported arguments: ${arg_UNPARSED_ARGUMENTS}") + endif() + + string(TOLOWER ${contentName} contentNameLower) + set(prefix "_FetchContent_${contentNameLower}") + + set(propertyName "${prefix}_sourceDir") + define_property( + GLOBAL + PROPERTY ${propertyName} + BRIEF_DOCS "Internal implementation detail of FetchContent_Populate()" + FULL_DOCS "Details used by FetchContent_Populate() for ${contentName}" + ) + set_property(GLOBAL PROPERTY ${propertyName} "${arg_SOURCE_DIR}") + + set(propertyName "${prefix}_binaryDir") + define_property( + GLOBAL + PROPERTY ${propertyName} + BRIEF_DOCS "Internal implementation detail of FetchContent_Populate()" + FULL_DOCS "Details used by FetchContent_Populate() for ${contentName}" + ) + set_property(GLOBAL PROPERTY ${propertyName} "${arg_BINARY_DIR}") + + set(propertyName "${prefix}_populated") + define_property( + GLOBAL + PROPERTY ${propertyName} + BRIEF_DOCS "Internal implementation detail of FetchContent_Populate()" + FULL_DOCS "Details used by FetchContent_Populate() for ${contentName}" + ) + set_property(GLOBAL PROPERTY ${propertyName} TRUE) +endfunction() + +# Download and add a package from source +function(CPMAddPackage) + cpm_set_policies() + + list(LENGTH ARGN argnLength) + if(argnLength EQUAL 1) + cpm_parse_add_package_single_arg("${ARGN}" ARGN) + + # The shorthand syntax implies EXCLUDE_FROM_ALL and SYSTEM + set(ARGN "${ARGN};EXCLUDE_FROM_ALL;YES;SYSTEM;YES;") + endif() + + set(oneValueArgs + NAME + FORCE + VERSION + GIT_TAG + DOWNLOAD_ONLY + GITHUB_REPOSITORY + GITLAB_REPOSITORY + BITBUCKET_REPOSITORY + GIT_REPOSITORY + SOURCE_DIR + FIND_PACKAGE_ARGUMENTS + NO_CACHE + SYSTEM + GIT_SHALLOW + EXCLUDE_FROM_ALL + SOURCE_SUBDIR + CUSTOM_CACHE_KEY + ) + + set(multiValueArgs URL OPTIONS DOWNLOAD_COMMAND PATCHES) + + cmake_parse_arguments(CPM_ARGS "" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}") + + # Set default values for arguments + + if(NOT DEFINED CPM_ARGS_VERSION) + if(DEFINED CPM_ARGS_GIT_TAG) + cpm_get_version_from_git_tag("${CPM_ARGS_GIT_TAG}" CPM_ARGS_VERSION) + endif() + endif() + + if(CPM_ARGS_DOWNLOAD_ONLY) + set(DOWNLOAD_ONLY ${CPM_ARGS_DOWNLOAD_ONLY}) + else() + set(DOWNLOAD_ONLY NO) + endif() + + if(DEFINED CPM_ARGS_GITHUB_REPOSITORY) + set(CPM_ARGS_GIT_REPOSITORY "https://github.com/${CPM_ARGS_GITHUB_REPOSITORY}.git") + elseif(DEFINED CPM_ARGS_GITLAB_REPOSITORY) + set(CPM_ARGS_GIT_REPOSITORY "https://gitlab.com/${CPM_ARGS_GITLAB_REPOSITORY}.git") + elseif(DEFINED CPM_ARGS_BITBUCKET_REPOSITORY) + set(CPM_ARGS_GIT_REPOSITORY "https://bitbucket.org/${CPM_ARGS_BITBUCKET_REPOSITORY}.git") + endif() + + if(DEFINED CPM_ARGS_GIT_REPOSITORY) + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS GIT_REPOSITORY ${CPM_ARGS_GIT_REPOSITORY}) + if(NOT DEFINED CPM_ARGS_GIT_TAG) + set(CPM_ARGS_GIT_TAG v${CPM_ARGS_VERSION}) + endif() + + # If a name wasn't provided, try to infer it from the git repo + if(NOT DEFINED CPM_ARGS_NAME) + cpm_package_name_from_git_uri(${CPM_ARGS_GIT_REPOSITORY} CPM_ARGS_NAME) + endif() + endif() + + set(CPM_SKIP_FETCH FALSE) + + if(DEFINED CPM_ARGS_GIT_TAG) + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS GIT_TAG ${CPM_ARGS_GIT_TAG}) + # If GIT_SHALLOW is explicitly specified, honor the value. + if(DEFINED CPM_ARGS_GIT_SHALLOW) + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS GIT_SHALLOW ${CPM_ARGS_GIT_SHALLOW}) + endif() + endif() + + if(DEFINED CPM_ARGS_URL) + # If a name or version aren't provided, try to infer them from the URL + list(GET CPM_ARGS_URL 0 firstUrl) + cpm_package_name_and_ver_from_url(${firstUrl} nameFromUrl verFromUrl) + # If we fail to obtain name and version from the first URL, we could try other URLs if any. + # However multiple URLs are expected to be quite rare, so for now we won't bother. + + # If the caller provided their own name and version, they trump the inferred ones. + if(NOT DEFINED CPM_ARGS_NAME) + set(CPM_ARGS_NAME ${nameFromUrl}) + endif() + if(NOT DEFINED CPM_ARGS_VERSION) + set(CPM_ARGS_VERSION ${verFromUrl}) + endif() + + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS URL "${CPM_ARGS_URL}") + endif() + + # Check for required arguments + + if(NOT DEFINED CPM_ARGS_NAME) + message( + FATAL_ERROR + "${CPM_INDENT} 'NAME' was not provided and couldn't be automatically inferred for package added with arguments: '${ARGN}'" + ) + endif() + + # Check if package has been added before + cpm_check_if_package_already_added(${CPM_ARGS_NAME} "${CPM_ARGS_VERSION}") + if(CPM_PACKAGE_ALREADY_ADDED) + cpm_export_variables(${CPM_ARGS_NAME}) + return() + endif() + + # Check for manual overrides + if(NOT CPM_ARGS_FORCE AND NOT "${CPM_${CPM_ARGS_NAME}_SOURCE}" STREQUAL "") + set(PACKAGE_SOURCE ${CPM_${CPM_ARGS_NAME}_SOURCE}) + set(CPM_${CPM_ARGS_NAME}_SOURCE "") + CPMAddPackage( + NAME "${CPM_ARGS_NAME}" + SOURCE_DIR "${PACKAGE_SOURCE}" + EXCLUDE_FROM_ALL "${CPM_ARGS_EXCLUDE_FROM_ALL}" + SYSTEM "${CPM_ARGS_SYSTEM}" + PATCHES "${CPM_ARGS_PATCHES}" + OPTIONS "${CPM_ARGS_OPTIONS}" + SOURCE_SUBDIR "${CPM_ARGS_SOURCE_SUBDIR}" + DOWNLOAD_ONLY "${DOWNLOAD_ONLY}" + FORCE True + ) + cpm_export_variables(${CPM_ARGS_NAME}) + return() + endif() + + # Check for available declaration + if(NOT CPM_ARGS_FORCE AND NOT "${CPM_DECLARATION_${CPM_ARGS_NAME}}" STREQUAL "") + set(declaration ${CPM_DECLARATION_${CPM_ARGS_NAME}}) + set(CPM_DECLARATION_${CPM_ARGS_NAME} "") + CPMAddPackage(${declaration}) + cpm_export_variables(${CPM_ARGS_NAME}) + # checking again to ensure version and option compatibility + cpm_check_if_package_already_added(${CPM_ARGS_NAME} "${CPM_ARGS_VERSION}") + return() + endif() + + if(NOT CPM_ARGS_FORCE) + if(CPM_USE_LOCAL_PACKAGES OR CPM_LOCAL_PACKAGES_ONLY) + cpm_find_package(${CPM_ARGS_NAME} "${CPM_ARGS_VERSION}" ${CPM_ARGS_FIND_PACKAGE_ARGUMENTS}) + + if(CPM_PACKAGE_FOUND) + cpm_export_variables(${CPM_ARGS_NAME}) + return() + endif() + + if(CPM_LOCAL_PACKAGES_ONLY) + message( + SEND_ERROR + "${CPM_INDENT} ${CPM_ARGS_NAME} not found via find_package(${CPM_ARGS_NAME} ${CPM_ARGS_VERSION})" + ) + endif() + endif() + endif() + + CPMRegisterPackage("${CPM_ARGS_NAME}" "${CPM_ARGS_VERSION}") + + if(DEFINED CPM_ARGS_GIT_TAG) + set(PACKAGE_INFO "${CPM_ARGS_GIT_TAG}") + elseif(DEFINED CPM_ARGS_SOURCE_DIR) + set(PACKAGE_INFO "${CPM_ARGS_SOURCE_DIR}") + else() + set(PACKAGE_INFO "${CPM_ARGS_VERSION}") + endif() + + if(DEFINED FETCHCONTENT_BASE_DIR) + # respect user's FETCHCONTENT_BASE_DIR if set + set(CPM_FETCHCONTENT_BASE_DIR ${FETCHCONTENT_BASE_DIR}) + else() + set(CPM_FETCHCONTENT_BASE_DIR ${CMAKE_BINARY_DIR}/_deps) + endif() + + cpm_add_patches(${CPM_ARGS_PATCHES}) + + if(DEFINED CPM_ARGS_DOWNLOAD_COMMAND) + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS DOWNLOAD_COMMAND ${CPM_ARGS_DOWNLOAD_COMMAND}) + elseif(DEFINED CPM_ARGS_SOURCE_DIR) + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS SOURCE_DIR ${CPM_ARGS_SOURCE_DIR}) + if(NOT IS_ABSOLUTE ${CPM_ARGS_SOURCE_DIR}) + # Expand `CPM_ARGS_SOURCE_DIR` relative path. This is important because EXISTS doesn't work + # for relative paths. + get_filename_component( + source_directory ${CPM_ARGS_SOURCE_DIR} REALPATH BASE_DIR ${CMAKE_CURRENT_BINARY_DIR} + ) + else() + set(source_directory ${CPM_ARGS_SOURCE_DIR}) + endif() + if(NOT EXISTS ${source_directory}) + string(TOLOWER ${CPM_ARGS_NAME} lower_case_name) + # remove timestamps so CMake will re-download the dependency + file(REMOVE_RECURSE "${CPM_FETCHCONTENT_BASE_DIR}/${lower_case_name}-subbuild") + endif() + elseif(CPM_SOURCE_CACHE AND NOT CPM_ARGS_NO_CACHE) + string(TOLOWER ${CPM_ARGS_NAME} lower_case_name) + set(origin_parameters ${CPM_ARGS_UNPARSED_ARGUMENTS}) + list(SORT origin_parameters) + if(CPM_ARGS_CUSTOM_CACHE_KEY) + # Application set a custom unique directory name + set(download_directory ${CPM_SOURCE_CACHE}/${lower_case_name}/${CPM_ARGS_CUSTOM_CACHE_KEY}) + elseif(CPM_USE_NAMED_CACHE_DIRECTORIES) + string(SHA1 origin_hash "${origin_parameters};NEW_CACHE_STRUCTURE_TAG") + set(download_directory ${CPM_SOURCE_CACHE}/${lower_case_name}/${origin_hash}/${CPM_ARGS_NAME}) + else() + string(SHA1 origin_hash "${origin_parameters}") + set(download_directory ${CPM_SOURCE_CACHE}/${lower_case_name}/${origin_hash}) + endif() + # Expand `download_directory` relative path. This is important because EXISTS doesn't work for + # relative paths. + get_filename_component(download_directory ${download_directory} ABSOLUTE) + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS SOURCE_DIR ${download_directory}) + + if(CPM_SOURCE_CACHE) + file(LOCK ${download_directory}/../cmake.lock) + endif() + + if(EXISTS ${download_directory}) + if(CPM_SOURCE_CACHE) + file(LOCK ${download_directory}/../cmake.lock RELEASE) + endif() + + cpm_store_fetch_properties( + ${CPM_ARGS_NAME} "${download_directory}" + "${CPM_FETCHCONTENT_BASE_DIR}/${lower_case_name}-build" + ) + cpm_get_fetch_properties("${CPM_ARGS_NAME}") + + if(DEFINED CPM_ARGS_GIT_TAG AND NOT (PATCH_COMMAND IN_LIST CPM_ARGS_UNPARSED_ARGUMENTS)) + # warn if cache has been changed since checkout + cpm_check_git_working_dir_is_clean(${download_directory} ${CPM_ARGS_GIT_TAG} IS_CLEAN) + if(NOT ${IS_CLEAN}) + message( + WARNING "${CPM_INDENT} Cache for ${CPM_ARGS_NAME} (${download_directory}) is dirty" + ) + endif() + endif() + + cpm_add_subdirectory( + "${CPM_ARGS_NAME}" + "${DOWNLOAD_ONLY}" + "${${CPM_ARGS_NAME}_SOURCE_DIR}/${CPM_ARGS_SOURCE_SUBDIR}" + "${${CPM_ARGS_NAME}_BINARY_DIR}" + "${CPM_ARGS_EXCLUDE_FROM_ALL}" + "${CPM_ARGS_SYSTEM}" + "${CPM_ARGS_OPTIONS}" + ) + set(PACKAGE_INFO "${PACKAGE_INFO} at ${download_directory}") + + # As the source dir is already cached/populated, we override the call to FetchContent. + set(CPM_SKIP_FETCH TRUE) + cpm_override_fetchcontent( + "${lower_case_name}" SOURCE_DIR "${${CPM_ARGS_NAME}_SOURCE_DIR}/${CPM_ARGS_SOURCE_SUBDIR}" + BINARY_DIR "${${CPM_ARGS_NAME}_BINARY_DIR}" + ) + + else() + # Enable shallow clone when GIT_TAG is not a commit hash. Our guess may not be accurate, but + # it should guarantee no commit hash get mis-detected. + if(NOT DEFINED CPM_ARGS_GIT_SHALLOW) + cpm_is_git_tag_commit_hash("${CPM_ARGS_GIT_TAG}" IS_HASH) + if(NOT ${IS_HASH}) + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS GIT_SHALLOW TRUE) + endif() + endif() + + # remove timestamps so CMake will re-download the dependency + file(REMOVE_RECURSE ${CPM_FETCHCONTENT_BASE_DIR}/${lower_case_name}-subbuild) + set(PACKAGE_INFO "${PACKAGE_INFO} to ${download_directory}") + endif() + endif() + + if(NOT "${DOWNLOAD_ONLY}") + cpm_create_module_file(${CPM_ARGS_NAME} "CPMAddPackage(\"${ARGN}\")") + endif() + + if(CPM_PACKAGE_LOCK_ENABLED) + if((CPM_ARGS_VERSION AND NOT CPM_ARGS_SOURCE_DIR) OR CPM_INCLUDE_ALL_IN_PACKAGE_LOCK) + cpm_add_to_package_lock(${CPM_ARGS_NAME} "${ARGN}") + elseif(CPM_ARGS_SOURCE_DIR) + cpm_add_comment_to_package_lock(${CPM_ARGS_NAME} "local directory") + else() + cpm_add_comment_to_package_lock(${CPM_ARGS_NAME} "${ARGN}") + endif() + endif() + + cpm_message( + STATUS "${CPM_INDENT} Adding package ${CPM_ARGS_NAME}@${CPM_ARGS_VERSION} (${PACKAGE_INFO})" + ) + + if(NOT CPM_SKIP_FETCH) + # CMake 3.28 added EXCLUDE, SYSTEM (3.25), and SOURCE_SUBDIR (3.18) to FetchContent_Declare. + # Calling FetchContent_MakeAvailable will then internally forward these options to + # add_subdirectory. Up until these changes, we had to call FetchContent_Populate and + # add_subdirectory separately, which is no longer necessary and has been deprecated as of 3.30. + # A Bug in CMake prevents us to use the non-deprecated functions until 3.30.3. + set(fetchContentDeclareExtraArgs "") + if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.30.3") + if(${CPM_ARGS_EXCLUDE_FROM_ALL}) + list(APPEND fetchContentDeclareExtraArgs EXCLUDE_FROM_ALL) + endif() + if(${CPM_ARGS_SYSTEM}) + list(APPEND fetchContentDeclareExtraArgs SYSTEM) + endif() + if(DEFINED CPM_ARGS_SOURCE_SUBDIR) + list(APPEND fetchContentDeclareExtraArgs SOURCE_SUBDIR ${CPM_ARGS_SOURCE_SUBDIR}) + endif() + # For CMake version <3.28 OPTIONS are parsed in cpm_add_subdirectory + if(CPM_ARGS_OPTIONS AND NOT DOWNLOAD_ONLY) + foreach(OPTION ${CPM_ARGS_OPTIONS}) + cpm_parse_option("${OPTION}") + set(${OPTION_KEY} "${OPTION_VALUE}") + endforeach() + endif() + endif() + cpm_declare_fetch( + "${CPM_ARGS_NAME}" ${fetchContentDeclareExtraArgs} "${CPM_ARGS_UNPARSED_ARGUMENTS}" + ) + + cpm_fetch_package("${CPM_ARGS_NAME}" ${DOWNLOAD_ONLY} populated ${CPM_ARGS_UNPARSED_ARGUMENTS}) + if(CPM_SOURCE_CACHE AND download_directory) + file(LOCK ${download_directory}/../cmake.lock RELEASE) + endif() + if(${populated} AND ${CMAKE_VERSION} VERSION_LESS "3.30.3") + cpm_add_subdirectory( + "${CPM_ARGS_NAME}" + "${DOWNLOAD_ONLY}" + "${${CPM_ARGS_NAME}_SOURCE_DIR}/${CPM_ARGS_SOURCE_SUBDIR}" + "${${CPM_ARGS_NAME}_BINARY_DIR}" + "${CPM_ARGS_EXCLUDE_FROM_ALL}" + "${CPM_ARGS_SYSTEM}" + "${CPM_ARGS_OPTIONS}" + ) + endif() + cpm_get_fetch_properties("${CPM_ARGS_NAME}") + endif() + + set(${CPM_ARGS_NAME}_ADDED YES) + cpm_export_variables("${CPM_ARGS_NAME}") +endfunction() + +# Fetch a previously declared package +macro(CPMGetPackage Name) + if(DEFINED "CPM_DECLARATION_${Name}") + CPMAddPackage(NAME ${Name}) + else() + message(SEND_ERROR "${CPM_INDENT} Cannot retrieve package ${Name}: no declaration available") + endif() +endmacro() + +# export variables available to the caller to the parent scope expects ${CPM_ARGS_NAME} to be set +macro(cpm_export_variables name) + set(${name}_SOURCE_DIR + "${${name}_SOURCE_DIR}" + PARENT_SCOPE + ) + set(${name}_BINARY_DIR + "${${name}_BINARY_DIR}" + PARENT_SCOPE + ) + set(${name}_ADDED + "${${name}_ADDED}" + PARENT_SCOPE + ) + set(CPM_LAST_PACKAGE_NAME + "${name}" + PARENT_SCOPE + ) +endmacro() + +# declares a package, so that any call to CPMAddPackage for the package name will use these +# arguments instead. Previous declarations will not be overridden. +macro(CPMDeclarePackage Name) + if(NOT DEFINED "CPM_DECLARATION_${Name}") + set("CPM_DECLARATION_${Name}" "${ARGN}") + endif() +endmacro() + +function(cpm_add_to_package_lock Name) + if(NOT CPM_DONT_CREATE_PACKAGE_LOCK) + cpm_prettify_package_arguments(PRETTY_ARGN false ${ARGN}) + file(APPEND ${CPM_PACKAGE_LOCK_FILE} "# ${Name}\nCPMDeclarePackage(${Name}\n${PRETTY_ARGN})\n") + endif() +endfunction() + +function(cpm_add_comment_to_package_lock Name) + if(NOT CPM_DONT_CREATE_PACKAGE_LOCK) + cpm_prettify_package_arguments(PRETTY_ARGN true ${ARGN}) + file(APPEND ${CPM_PACKAGE_LOCK_FILE} + "# ${Name} (unversioned)\n# CPMDeclarePackage(${Name}\n${PRETTY_ARGN}#)\n" + ) + endif() +endfunction() + +# includes the package lock file if it exists and creates a target `cpm-update-package-lock` to +# update it +macro(CPMUsePackageLock file) + if(NOT CPM_DONT_CREATE_PACKAGE_LOCK) + get_filename_component(CPM_ABSOLUTE_PACKAGE_LOCK_PATH ${file} ABSOLUTE) + if(EXISTS ${CPM_ABSOLUTE_PACKAGE_LOCK_PATH}) + include(${CPM_ABSOLUTE_PACKAGE_LOCK_PATH}) + endif() + if(NOT TARGET cpm-update-package-lock) + add_custom_target( + cpm-update-package-lock COMMAND ${CMAKE_COMMAND} -E copy ${CPM_PACKAGE_LOCK_FILE} + ${CPM_ABSOLUTE_PACKAGE_LOCK_PATH} + ) + endif() + set(CPM_PACKAGE_LOCK_ENABLED true) + endif() +endmacro() + +# registers a package that has been added to CPM +function(CPMRegisterPackage PACKAGE VERSION) + list(APPEND CPM_PACKAGES ${PACKAGE}) + set(CPM_PACKAGES + ${CPM_PACKAGES} + CACHE INTERNAL "" + ) + set("CPM_PACKAGE_${PACKAGE}_VERSION" + ${VERSION} + CACHE INTERNAL "" + ) +endfunction() + +# retrieve the current version of the package to ${OUTPUT} +function(CPMGetPackageVersion PACKAGE OUTPUT) + set(${OUTPUT} + "${CPM_PACKAGE_${PACKAGE}_VERSION}" + PARENT_SCOPE + ) +endfunction() + +# declares a package in FetchContent_Declare +function(cpm_declare_fetch PACKAGE) + if(${CPM_DRY_RUN}) + cpm_message(STATUS "${CPM_INDENT} Package not declared (dry run)") + return() + endif() + + FetchContent_Declare(${PACKAGE} ${ARGN}) +endfunction() + +# returns properties for a package previously defined by cpm_declare_fetch +function(cpm_get_fetch_properties PACKAGE) + if(${CPM_DRY_RUN}) + return() + endif() + + set(${PACKAGE}_SOURCE_DIR + "${CPM_PACKAGE_${PACKAGE}_SOURCE_DIR}" + PARENT_SCOPE + ) + set(${PACKAGE}_BINARY_DIR + "${CPM_PACKAGE_${PACKAGE}_BINARY_DIR}" + PARENT_SCOPE + ) +endfunction() + +function(cpm_store_fetch_properties PACKAGE source_dir binary_dir) + if(${CPM_DRY_RUN}) + return() + endif() + + set(CPM_PACKAGE_${PACKAGE}_SOURCE_DIR + "${source_dir}" + CACHE INTERNAL "" + ) + set(CPM_PACKAGE_${PACKAGE}_BINARY_DIR + "${binary_dir}" + CACHE INTERNAL "" + ) +endfunction() + +# adds a package as a subdirectory if viable, according to provided options +function( + cpm_add_subdirectory + PACKAGE + DOWNLOAD_ONLY + SOURCE_DIR + BINARY_DIR + EXCLUDE + SYSTEM + OPTIONS +) + + if(NOT DOWNLOAD_ONLY AND EXISTS ${SOURCE_DIR}/CMakeLists.txt) + set(addSubdirectoryExtraArgs "") + if(EXCLUDE) + list(APPEND addSubdirectoryExtraArgs EXCLUDE_FROM_ALL) + endif() + if("${SYSTEM}" AND "${CMAKE_VERSION}" VERSION_GREATER_EQUAL "3.25") + # https://cmake.org/cmake/help/latest/prop_dir/SYSTEM.html#prop_dir:SYSTEM + list(APPEND addSubdirectoryExtraArgs SYSTEM) + endif() + if(OPTIONS) + foreach(OPTION ${OPTIONS}) + cpm_parse_option("${OPTION}") + set(${OPTION_KEY} "${OPTION_VALUE}") + endforeach() + endif() + set(CPM_OLD_INDENT "${CPM_INDENT}") + set(CPM_INDENT "${CPM_INDENT} ${PACKAGE}:") + add_subdirectory(${SOURCE_DIR} ${BINARY_DIR} ${addSubdirectoryExtraArgs}) + set(CPM_INDENT "${CPM_OLD_INDENT}") + endif() +endfunction() + +# downloads a previously declared package via FetchContent and exports the variables +# `${PACKAGE}_SOURCE_DIR` and `${PACKAGE}_BINARY_DIR` to the parent scope +function(cpm_fetch_package PACKAGE DOWNLOAD_ONLY populated) + set(${populated} + FALSE + PARENT_SCOPE + ) + if(${CPM_DRY_RUN}) + cpm_message(STATUS "${CPM_INDENT} Package ${PACKAGE} not fetched (dry run)") + return() + endif() + + FetchContent_GetProperties(${PACKAGE}) + + string(TOLOWER "${PACKAGE}" lower_case_name) + + if(NOT ${lower_case_name}_POPULATED) + if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.30.3") + if(DOWNLOAD_ONLY) + # MakeAvailable will call add_subdirectory internally which is not what we want when + # DOWNLOAD_ONLY is set. Populate will only download the dependency without adding it to the + # build + FetchContent_Populate( + ${PACKAGE} + SOURCE_DIR "${CPM_FETCHCONTENT_BASE_DIR}/${lower_case_name}-src" + BINARY_DIR "${CPM_FETCHCONTENT_BASE_DIR}/${lower_case_name}-build" + SUBBUILD_DIR "${CPM_FETCHCONTENT_BASE_DIR}/${lower_case_name}-subbuild" + ${ARGN} + ) + else() + FetchContent_MakeAvailable(${PACKAGE}) + endif() + else() + FetchContent_Populate(${PACKAGE}) + endif() + set(${populated} + TRUE + PARENT_SCOPE + ) + endif() + + cpm_store_fetch_properties( + ${CPM_ARGS_NAME} ${${lower_case_name}_SOURCE_DIR} ${${lower_case_name}_BINARY_DIR} + ) + + set(${PACKAGE}_SOURCE_DIR + ${${lower_case_name}_SOURCE_DIR} + PARENT_SCOPE + ) + set(${PACKAGE}_BINARY_DIR + ${${lower_case_name}_BINARY_DIR} + PARENT_SCOPE + ) +endfunction() + +# splits a package option +function(cpm_parse_option OPTION) + string(REGEX MATCH "^[^ ]+" OPTION_KEY "${OPTION}") + string(LENGTH "${OPTION}" OPTION_LENGTH) + string(LENGTH "${OPTION_KEY}" OPTION_KEY_LENGTH) + if(OPTION_KEY_LENGTH STREQUAL OPTION_LENGTH) + # no value for key provided, assume user wants to set option to "ON" + set(OPTION_VALUE "ON") + else() + math(EXPR OPTION_KEY_LENGTH "${OPTION_KEY_LENGTH}+1") + string(SUBSTRING "${OPTION}" "${OPTION_KEY_LENGTH}" "-1" OPTION_VALUE) + endif() + set(OPTION_KEY + "${OPTION_KEY}" + PARENT_SCOPE + ) + set(OPTION_VALUE + "${OPTION_VALUE}" + PARENT_SCOPE + ) +endfunction() + +# guesses the package version from a git tag +function(cpm_get_version_from_git_tag GIT_TAG RESULT) + string(LENGTH ${GIT_TAG} length) + if(length EQUAL 40) + # GIT_TAG is probably a git hash + set(${RESULT} + 0 + PARENT_SCOPE + ) + else() + string(REGEX MATCH "v?([0123456789.]*).*" _ ${GIT_TAG}) + set(${RESULT} + ${CMAKE_MATCH_1} + PARENT_SCOPE + ) + endif() +endfunction() + +# guesses if the git tag is a commit hash or an actual tag or a branch name. +function(cpm_is_git_tag_commit_hash GIT_TAG RESULT) + string(LENGTH "${GIT_TAG}" length) + # full hash has 40 characters, and short hash has at least 7 characters. + if(length LESS 7 OR length GREATER 40) + set(${RESULT} + 0 + PARENT_SCOPE + ) + else() + if(${GIT_TAG} MATCHES "^[a-fA-F0-9]+$") + set(${RESULT} + 1 + PARENT_SCOPE + ) + else() + set(${RESULT} + 0 + PARENT_SCOPE + ) + endif() + endif() +endfunction() + +function(cpm_prettify_package_arguments OUT_VAR IS_IN_COMMENT) + set(oneValueArgs + NAME + FORCE + VERSION + GIT_TAG + DOWNLOAD_ONLY + GITHUB_REPOSITORY + GITLAB_REPOSITORY + BITBUCKET_REPOSITORY + GIT_REPOSITORY + SOURCE_DIR + FIND_PACKAGE_ARGUMENTS + NO_CACHE + SYSTEM + GIT_SHALLOW + EXCLUDE_FROM_ALL + SOURCE_SUBDIR + ) + set(multiValueArgs URL OPTIONS DOWNLOAD_COMMAND) + cmake_parse_arguments(CPM_ARGS "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + foreach(oneArgName ${oneValueArgs}) + if(DEFINED CPM_ARGS_${oneArgName}) + if(${IS_IN_COMMENT}) + string(APPEND PRETTY_OUT_VAR "#") + endif() + if(${oneArgName} STREQUAL "SOURCE_DIR") + string(REPLACE ${CMAKE_SOURCE_DIR} "\${CMAKE_SOURCE_DIR}" CPM_ARGS_${oneArgName} + ${CPM_ARGS_${oneArgName}} + ) + endif() + string(APPEND PRETTY_OUT_VAR " ${oneArgName} ${CPM_ARGS_${oneArgName}}\n") + endif() + endforeach() + foreach(multiArgName ${multiValueArgs}) + if(DEFINED CPM_ARGS_${multiArgName}) + if(${IS_IN_COMMENT}) + string(APPEND PRETTY_OUT_VAR "#") + endif() + string(APPEND PRETTY_OUT_VAR " ${multiArgName}\n") + foreach(singleOption ${CPM_ARGS_${multiArgName}}) + if(${IS_IN_COMMENT}) + string(APPEND PRETTY_OUT_VAR "#") + endif() + string(APPEND PRETTY_OUT_VAR " \"${singleOption}\"\n") + endforeach() + endif() + endforeach() + + if(NOT "${CPM_ARGS_UNPARSED_ARGUMENTS}" STREQUAL "") + if(${IS_IN_COMMENT}) + string(APPEND PRETTY_OUT_VAR "#") + endif() + string(APPEND PRETTY_OUT_VAR " ") + foreach(CPM_ARGS_UNPARSED_ARGUMENT ${CPM_ARGS_UNPARSED_ARGUMENTS}) + string(APPEND PRETTY_OUT_VAR " ${CPM_ARGS_UNPARSED_ARGUMENT}") + endforeach() + string(APPEND PRETTY_OUT_VAR "\n") + endif() + + set(${OUT_VAR} + ${PRETTY_OUT_VAR} + PARENT_SCOPE + ) + +endfunction() diff --git a/xenium/acquire_guard.hpp b/include/xenium/acquire_guard.hpp similarity index 96% rename from xenium/acquire_guard.hpp rename to include/xenium/acquire_guard.hpp index 6fcef8f..350cf6a 100644 --- a/xenium/acquire_guard.hpp +++ b/include/xenium/acquire_guard.hpp @@ -1,29 +1,29 @@ -// -// Copyright (c) 2018-2020 Manuel Pöter. -// Licensed under the MIT License. See LICENSE file in the project root for full license information. -// - -#ifndef XENIUM_ACQUIRE_GUARD_HPP -#define XENIUM_ACQUIRE_GUARD_HPP - -#include - -namespace xenium { - -/** - * @brief Helper function to acquire a `guard_ptr` without having to define the `guard_ptr` - * instance beforehand. - * @param p the `concurrent_ptr` from which we want to acquire a safe reference. - * @param order the `memory_order` that shall be used in the `guard_ptr::acquire` operation. - * @return a `guard_ptr` to the node that `p` references. - */ -template -auto acquire_guard(ConcurrentPtr& p, std::memory_order order = std::memory_order_seq_cst) { - typename ConcurrentPtr::guard_ptr guard; - guard.acquire(p, order); - return guard; -} - -} // namespace xenium - -#endif +// +// Copyright (c) 2018-2020 Manuel Pöter. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +#ifndef XENIUM_ACQUIRE_GUARD_HPP +#define XENIUM_ACQUIRE_GUARD_HPP + +#include + +namespace xenium { + +/** + * @brief Helper function to acquire a `guard_ptr` without having to define the `guard_ptr` + * instance beforehand. + * @param p the `concurrent_ptr` from which we want to acquire a safe reference. + * @param order the `memory_order` that shall be used in the `guard_ptr::acquire` operation. + * @return a `guard_ptr` to the node that `p` references. + */ +template +auto acquire_guard(ConcurrentPtr& p, std::memory_order order = std::memory_order_seq_cst) { + typename ConcurrentPtr::guard_ptr guard; + guard.acquire(p, order); + return guard; +} + +} // namespace xenium + +#endif diff --git a/xenium/aligned_object.hpp b/include/xenium/aligned_object.hpp similarity index 97% rename from xenium/aligned_object.hpp rename to include/xenium/aligned_object.hpp index 2fa84c4..026ccb9 100644 --- a/xenium/aligned_object.hpp +++ b/include/xenium/aligned_object.hpp @@ -1,40 +1,40 @@ -// -// Copyright (c) 2018-2020 Manuel Pöter. -// Licensed under the MIT License. See LICENSE file in the project root for full license information. -// - -#ifndef XENIUM_ALIGNED_OBJECT_HPP -#define XENIUM_ALIGNED_OBJECT_HPP - -#include -#include - -namespace xenium { - -/** - * @brief A small helper class for correctly aligned dynamic allocations of - * over-aligned types. - * - * Dynamic allocation using the standard `operator new` does not consider the - * alignment of the allocated type. To ensure that dynamic allocations are - * correctly aligned, classes can inherit from `aligned_object`. - * If not specified explicitly, `aligned_object` implicitly uses the alignment - * from `std::alignment_of()`. - * @tparam Derived - * @tparam Alignment the alignment to use for dynamic allocations. - * If this value is 0, the alignment is derived from `Derived`. Defaults to 0. - */ -template -struct aligned_object { - static void* operator new(size_t sz) { return ::operator new(sz, alignment()); } - - static void operator delete(void* p) { ::operator delete(p, alignment()); } - -private: - static constexpr std::align_val_t alignment() { - return static_cast(Alignment == 0 ? std::alignment_of() : Alignment); - } -}; -} // namespace xenium - -#endif +// +// Copyright (c) 2018-2020 Manuel Pöter. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +#ifndef XENIUM_ALIGNED_OBJECT_HPP +#define XENIUM_ALIGNED_OBJECT_HPP + +#include +#include + +namespace xenium { + +/** + * @brief A small helper class for correctly aligned dynamic allocations of + * over-aligned types. + * + * Dynamic allocation using the standard `operator new` does not consider the + * alignment of the allocated type. To ensure that dynamic allocations are + * correctly aligned, classes can inherit from `aligned_object`. + * If not specified explicitly, `aligned_object` implicitly uses the alignment + * from `std::alignment_of()`. + * @tparam Derived + * @tparam Alignment the alignment to use for dynamic allocations. + * If this value is 0, the alignment is derived from `Derived`. Defaults to 0. + */ +template +struct aligned_object { + static void* operator new(size_t sz) { return ::operator new(sz, alignment()); } + + static void operator delete(void* p) { ::operator delete(p, alignment()); } + +private: + static constexpr std::align_val_t alignment() { + return static_cast(Alignment == 0 ? std::alignment_of() : Alignment); + } +}; +} // namespace xenium + +#endif diff --git a/xenium/backoff.hpp b/include/xenium/backoff.hpp similarity index 95% rename from xenium/backoff.hpp rename to include/xenium/backoff.hpp index 4efd15c..4d7964d 100644 --- a/xenium/backoff.hpp +++ b/include/xenium/backoff.hpp @@ -1,44 +1,44 @@ -// -// Copyright (c) 2018-2020 Manuel Pöter. -// Licensed under the MIT License. See LICENSE file in the project root for full license information. -// - -#ifndef XENIUM_BACKOFF_HPP -#define XENIUM_BACKOFF_HPP - -#include -#include - -namespace xenium { -/** - * @brief Dummy backoff strategy that does nothing. - */ -struct no_backoff { - void operator()() {} -}; - -/** - * @brief Simple backoff strategy that always perfoms a single `hardware_pause` operation. - */ -struct single_backoff { - void operator()() { detail::hardware_pause(); } -}; - -template -struct exponential_backoff { - static_assert(Max > 0, "Max must be greater than zero. If you don't want to backoff use the `no_backoff` class."); - - void operator()() { - for (unsigned i = 0; i < count; ++i) { - detail::hardware_pause(); - } - count = std::min(Max, count * 2); - } - -private: - unsigned count = 1; -}; - -} // namespace xenium - -#endif +// +// Copyright (c) 2018-2020 Manuel Pöter. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +#ifndef XENIUM_BACKOFF_HPP +#define XENIUM_BACKOFF_HPP + +#include +#include + +namespace xenium { +/** + * @brief Dummy backoff strategy that does nothing. + */ +struct no_backoff { + void operator()() {} +}; + +/** + * @brief Simple backoff strategy that always perfoms a single `hardware_pause` operation. + */ +struct single_backoff { + void operator()() { detail::hardware_pause(); } +}; + +template +struct exponential_backoff { + static_assert(Max > 0, "Max must be greater than zero. If you don't want to backoff use the `no_backoff` class."); + + void operator()() { + for (unsigned i = 0; i < count; ++i) { + detail::hardware_pause(); + } + count = std::min(Max, count * 2); + } + +private: + unsigned count = 1; +}; + +} // namespace xenium + +#endif diff --git a/xenium/chase_work_stealing_deque.hpp b/include/xenium/chase_work_stealing_deque.hpp similarity index 100% rename from xenium/chase_work_stealing_deque.hpp rename to include/xenium/chase_work_stealing_deque.hpp diff --git a/xenium/detail/fixed_size_circular_array.hpp b/include/xenium/detail/fixed_size_circular_array.hpp similarity index 100% rename from xenium/detail/fixed_size_circular_array.hpp rename to include/xenium/detail/fixed_size_circular_array.hpp diff --git a/xenium/detail/growing_circular_array.hpp b/include/xenium/detail/growing_circular_array.hpp similarity index 100% rename from xenium/detail/growing_circular_array.hpp rename to include/xenium/detail/growing_circular_array.hpp diff --git a/xenium/detail/hardware.hpp b/include/xenium/detail/hardware.hpp similarity index 100% rename from xenium/detail/hardware.hpp rename to include/xenium/detail/hardware.hpp diff --git a/xenium/detail/nikolaev_scq.hpp b/include/xenium/detail/nikolaev_scq.hpp similarity index 100% rename from xenium/detail/nikolaev_scq.hpp rename to include/xenium/detail/nikolaev_scq.hpp diff --git a/xenium/detail/pointer_queue_traits.hpp b/include/xenium/detail/pointer_queue_traits.hpp similarity index 100% rename from xenium/detail/pointer_queue_traits.hpp rename to include/xenium/detail/pointer_queue_traits.hpp diff --git a/xenium/detail/port.hpp b/include/xenium/detail/port.hpp similarity index 100% rename from xenium/detail/port.hpp rename to include/xenium/detail/port.hpp diff --git a/xenium/harris_michael_hash_map.hpp b/include/xenium/harris_michael_hash_map.hpp similarity index 97% rename from xenium/harris_michael_hash_map.hpp rename to include/xenium/harris_michael_hash_map.hpp index c73d688..ef429c1 100644 --- a/xenium/harris_michael_hash_map.hpp +++ b/include/xenium/harris_michael_hash_map.hpp @@ -1,734 +1,734 @@ -// -// Copyright (c) 2018-2020 Manuel Pöter. -// Licensed under the MIT License. See LICENSE file in the project root for full license information. -// - -#ifndef XENIUM_HARRIS_MICHAEL_HASH_MAP_HPP -#define XENIUM_HARRIS_MICHAEL_HASH_MAP_HPP - -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace xenium { - -namespace policy { - /** - * @brief Policy to configure the number of buckets in `harris_michael_hash_map`. - * @tparam Value - */ - template - struct buckets; - - /** - * @brief Policy to configure the function that maps the hash value to a bucket - * in `harris_michael_hash_map`. - * - * This can be used to apply other multiplicative methods like fibonacci hashing. - * - * @tparam T - */ - template - struct map_to_bucket; - - /** - * @brief Policy to configure whether the hash value should be stored and used - * during lookup operations in `harris_michael_hash_map`. - * - * This can improve performance for complex types with expensive compare operations - * like strings. - * - * @tparam T - */ - template - struct memoize_hash; -} // namespace policy - -/** - * @brief A generic lock-free hash-map. - * - * This hash-map consists of a fixed number of buckets were each bucket is essentially - * a `harris_michael_list_based_set` instance. The number of buckets is fixed, so the - * hash-map does not support dynamic resizing. - * - * This hash-map is less efficient than many other available concurrent hash-maps, but it is - * lock-free and fully generic, i.e., it supports arbitrary types for `Key` and `Value`. - * - * This data structure is based on the solution proposed by Michael \[[Mic02](index.html#ref-michael-2002)\] - * which builds upon the original proposal by Harris \[[Har01](index.html#ref-harris-2001)\]. - * - * Supported policies: - * * `xenium::policy::reclaimer`
- * Defines the reclamation scheme to be used for internal nodes. (**required**) - * * `xenium::policy::hash`
- * Defines the hash function. (*optional*; defaults to `xenium::hash`) - * * `xenium::policy::map_to_bucket`
- * Defines the function that is used to map the calculated hash to a bucket. - * (*optional*; defaults to `xenium::utils::modulo`) - * * `xenium::policy::backoff`
- * Defines the backoff strategy. (*optional*; defaults to `xenium::no_backoff`) - * * `xenium::policy::buckets`
- * Defines the number of buckets. (*optional*; defaults to 512) - * * `xenium::policy::memoize_hash`
- * Defines whether the hash should be stored and used during lookup operations. - * (*optional*; defaults to false for scalar `Key` types; otherwise true) - * - * @tparam Key - * @tparam Value - * @tparam Policies list of policies to customize the behaviour - */ -template -class harris_michael_hash_map { -public: - using value_type = std::pair; - using reclaimer = parameter::type_param_t; - using hash = parameter::type_param_t, Policies...>; - using map_to_bucket = parameter::type_param_t, Policies...>; - using backoff = parameter::type_param_t; - static constexpr std::size_t num_buckets = - parameter::value_param_t::value; - static constexpr bool memoize_hash = - parameter::value_param_t::value, Policies...>::value; - - template - using with = harris_michael_hash_map; - - static_assert(parameter::is_set::value, "reclaimer policy must be specified"); - - class iterator; - class accessor; - - harris_michael_hash_map() = default; - ~harris_michael_hash_map(); - - /** - * @brief Inserts a new element into the container if the container doesn't already contain an - * element with an equivalent key. The element is constructed in-place with the given `args`. - * - * The element is always constructed. If there already is an element with the key in the container, - * the newly constructed element will be destroyed immediately. - * - * No iterators or references are invalidated. - * - * Progress guarantees: lock-free - * - * @param args arguments to forward to the constructor of the element - * @return `true` if an element was inserted, otherwise `false` - */ - template - bool emplace(Args&&... args); - - /** - * @brief Inserts a new element into the container if the container doesn't already contain an - * element with an equivalent key. The element is constructed in-place with the given `args`. - * - * The element is always constructed. If there already is an element with the key in the container, - * the newly constructed element will be destroyed immediately. - * - * No iterators or references are invalidated. - * - * Progress guarantees: lock-free - * - * @param args arguments to forward to the constructor of the element - * @return a pair consisting of an iterator to the inserted element, or the already-existing element - * if no insertion happened, and a bool denoting whether the insertion took place; - * `true` if an element was inserted, otherwise `false` - */ - template - std::pair emplace_or_get(Args&&... args); - - /** - * @brief Inserts a new element into the container if the container doesn't already contain an - * element with an equivalent key. The element is constructed as `value_type(std::piecewise_construct, - * std::forward_as_tuple(k), std::forward_as_tuple(std::forward(args)...))`. - * - * The element may be constructed even if there already is an element with the key in the container, - * in which case the newly constructed element will be destroyed immediately. - * - * No iterators or references are invalidated. - * Progress guarantees: lock-free - * - * @param key the key of element to be inserted. - * @param args arguments to forward to the constructor of the element - * @return a pair consisting of an iterator to the inserted element, or the already-existing element - * if no insertion happened, and a bool denoting whether the insertion took place; - * `true` if an element was inserted, otherwise `false` - */ - template - std::pair get_or_emplace(Key key, Args&&... args); - - /** - * @brief Inserts a new element into the container if the container doesn't already contain an - * element with an equivalent key. The value for the newly constructed element is created by - * calling `value_factory`. - * - * The element may be constructed even if there already is an element with the key in the container, - * in which case the newly constructed element will be destroyed immediately. - * - * No iterators or references are invalidated. - * Progress guarantees: lock-free - * - * @tparam Func - * @param key the key of element to be inserted. - * @param factory a functor that is used to create the `Value` instance when constructing - * the new element to be inserted. - * @return a pair consisting of an iterator to the inserted element, or the already-existing element - * if no insertion happened, and a bool denoting whether the insertion took place; - * `true` if an element was inserted, otherwise `false` - */ - template - std::pair get_or_emplace_lazy(Key key, Factory factory); - - /** - * @brief Removes the element with the key equivalent to key (if one exists). - * - * No iterators or references are invalidated. - * - * Progress guarantees: lock-free - * - * @param key key of the element to remove - * @return `true` if an element was removed, otherwise `false` - */ - bool erase(const Key& key); - - /** - * @brief Removes the specified element from the container. - * - * No iterators or references are invalidated. - * - * Progress guarantees: lock-free - * - * @param pos the iterator identifying the element to remove - * @return iterator following the last removed element - */ - iterator erase(iterator pos); - - /** - * @brief Finds an element with key equivalent to key. - * - * Progress guarantees: lock-free - * - * @param key key of the element to search for - * @return iterator to an element with key equivalent to key if such element is found, - * otherwise past-the-end iterator - */ - iterator find(const Key& key); - - /** - * @brief Checks if there is an element with key equivalent to key in the container. - * - * Progress guarantees: lock-free - * - * @param key key of the element to search for - * @return `true` if there is such an element, otherwise `false` - */ - bool contains(const Key& key); - - /** - * @brief - * - * The `accessor` - * @param key - * @return an accessor to the element's value - */ - accessor operator[](const Key& key); - - /** - * @brief Returns an iterator to the first element of the container. - * @return iterator to the first element - */ - iterator begin(); - - /** - * @brief Returns an iterator to the element following the last element of the container. - * - * This element acts as a placeholder; attempting to access it results in undefined behavior. - * @return iterator to the element following the last element. - */ - iterator end(); - -private: - struct node; - using hash_t = std::size_t; - using concurrent_ptr = typename reclaimer::template concurrent_ptr; - using marked_ptr = typename concurrent_ptr::marked_ptr; - using guard_ptr = typename concurrent_ptr::guard_ptr; - - template - std::pair do_get_or_emplace_lazy(Key key, Factory node_factory); - - struct construct_without_hash {}; - - struct data_without_hash { - value_type value; - template - explicit data_without_hash(hash_t /*hash*/, Args&&... args) : value(std::forward(args)...) {} - template - explicit data_without_hash(construct_without_hash, Args&&... args) : value(std::forward(args)...) {} - [[nodiscard]] hash_t get_hash() const { return hash{}(value.first); } - [[nodiscard]] bool greater_or_equal(hash_t /*h*/, const Key& key) const { return value.first >= key; } - }; - - struct data_with_hash { - hash_t hash; - value_type value; - template - explicit data_with_hash(hash_t hash, Args&&... args) : hash(hash), value(std::forward(args)...) {} - template - explicit data_with_hash(construct_without_hash, Args&&... args) : value(std::forward(args)...) { - hash = harris_michael_hash_map::hash{}(value.first); - } - [[nodiscard]] hash_t get_hash() const { return hash; } - [[nodiscard]] bool greater_or_equal(hash_t h, const Key& key) const { return hash >= h && value.first >= key; } - }; - - using data_t = std::conditional_t; - - struct node : reclaimer::template enable_concurrent_ptr { - data_t data; - concurrent_ptr next; - template - explicit node(Args&&... args) : data(std::forward(args)...), next() {} - }; - - struct find_info { - concurrent_ptr* prev; - marked_ptr next{}; - guard_ptr cur{}; - guard_ptr save{}; - }; - - bool find(hash_t hash, const Key& key, std::size_t bucket, find_info& info, backoff& backoff); - - concurrent_ptr buckets[num_buckets]; -}; - -/** - * @brief A ForwardIterator to safely iterate the hash-map. - * - * Iterators are not invalidated by concurrent insert/erase operations. However, conflicting erase - * operations can have a negative impact on the performance when advancing the iterator, because it - * may be necessary to rescan the bucket's list to find the next element. - * - * *Note:* This iterator class does *not* provide multi-pass guarantee as `a == b` does not imply `++a == ++b`. - * - * *Note:* Each iterator internally holds two `guard_ptr` instances. This has to be considered when using - * a reclamation scheme that requires per-instance resources like `hazard_pointer` or `hazard_eras`. - * It is therefore highly recommended to use prefix increments wherever possible. - */ -template -class harris_michael_hash_map::iterator { -public: - using iterator_category = std::forward_iterator_tag; - using value_type = harris_michael_hash_map::value_type; - using difference_type = std::ptrdiff_t; - using pointer = value_type*; - using reference = value_type&; - - iterator(iterator&&) = default; - iterator(const iterator&) = default; - - iterator& operator=(iterator&&) = default; - iterator& operator=(const iterator&) = default; - - iterator& operator++() { - assert(info.cur.get() != nullptr); - auto next = info.cur->next.load(std::memory_order_relaxed); - guard_ptr tmp_guard; - // (1) - this acquire-load synchronizes-with the release-CAS (8, 9, 10, 12, 15) - if (next.mark() == 0 && tmp_guard.acquire_if_equal(info.cur->next, next, std::memory_order_acquire)) { - info.prev = &info.cur->next; - info.save = std::move(info.cur); - info.cur = std::move(tmp_guard); - } else { - // cur is marked for removal - // -> use find to remove it and get to the next node with a key >= cur->key - // Note: we have to copy key here! - Key key = info.cur->data.value.first; - hash_t h = info.cur->data.get_hash(); - backoff backoff; - map->find(h, key, bucket, info, backoff); - } - assert(info.prev == &map->buckets[bucket] || info.cur.get() == nullptr || - (info.save.get() != nullptr && &info.save->next == info.prev)); - - if (!info.cur) { - move_to_next_bucket(); - } - - return *this; - } - iterator operator++(int) { - iterator retval = *this; - ++(*this); - return retval; - } - bool operator==(const iterator& other) const { return info.cur.get() == other.info.cur.get(); } - bool operator!=(const iterator& other) const { return !(*this == other); } - reference operator*() const noexcept { return info.cur->data.value; } - pointer operator->() const noexcept { return &info.cur->data.value; } - - void reset() { - bucket = num_buckets; - info.prev = nullptr; - info.cur.reset(); - info.save.reset(); - } - -private: - friend harris_michael_hash_map; - - explicit iterator(harris_michael_hash_map* map) : map(map), bucket(num_buckets) {} - - explicit iterator(harris_michael_hash_map* map, std::size_t bucket) : map(map), bucket(bucket) { - info.prev = &map->buckets[bucket]; - // (2) - this acquire-load synchronizes-with the release-CAS (8, 9, 10, 12, 15) - info.cur.acquire(*info.prev, std::memory_order_acquire); - - if (!info.cur) { - move_to_next_bucket(); - } - } - - explicit iterator(harris_michael_hash_map* map, std::size_t bucket, find_info&& info) : - map(map), - bucket(bucket), - info(std::move(info)) {} - - void move_to_next_bucket() { - info.save.reset(); - while (!info.cur && bucket < num_buckets - 1) { - ++bucket; - info.prev = &map->buckets[bucket]; - // (3) - this acquire-load synchronizes-with the release-CAS (8, 9, 10, 12, 15) - info.cur.acquire(*info.prev, std::memory_order_acquire); - } - } - - harris_michael_hash_map* map; - std::size_t bucket; - find_info info; -}; - -/** - * @brief An accessor to safely access the value of an element. - * - * It is used as result type of `operator[]` to provide a similar behaviour as the - * STL `map`/`unordered_map` classes. However, the accessor has to be dereferenced to - * access the underlying value (`operator*`). - * - * Note: an `accessor` instance internally holds a `guard_ptr` to the value's element. - */ -template -class harris_michael_hash_map::accessor { -public: - Value* operator->() const noexcept { return &guard->data.value.second; } - Value& operator*() const noexcept { return guard->data.value.second; } - void reset() { guard.reset(); } - -private: - explicit accessor(guard_ptr&& guard) : guard(std::move(guard)) {} - guard_ptr guard; - friend harris_michael_hash_map; -}; - -template -harris_michael_hash_map::~harris_michael_hash_map() { - for (std::size_t i = 0; i < num_buckets; ++i) { - // (4) - this acquire-load synchronizes-with the release-CAS (8, 9, 10, 12, 15) - auto p = buckets[i].load(std::memory_order_acquire); - while (p) { - // (5) - this acquire-load synchronizes-with the release-CAS (8, 9, 10, 12, 15) - auto next = p->next.load(std::memory_order_acquire); - delete p.get(); - p = next; - } - } -} - -template -bool harris_michael_hash_map::find(hash_t hash, - const Key& key, - std::size_t bucket, - find_info& info, - backoff& backoff) { - auto& head = buckets[bucket]; - assert((info.save == nullptr && info.prev == &head) || &info.save->next == info.prev); - concurrent_ptr* start = info.prev; - guard_ptr start_guard = info.save; // we have to keep a guard_ptr to prevent start's node from getting reclaimed. -retry: - info.prev = start; - info.save = start_guard; - info.next = info.prev->load(std::memory_order_relaxed); - if (info.next.mark() != 0) { - // our start node is marked for removal -> we have to restart from head - start = &head; - start_guard.reset(); - goto retry; - } - - for (;;) { - // (6) - this acquire-load synchronizes-with the release-CAS (8, 9, 10, 12, 15) - if (!info.cur.acquire_if_equal(*info.prev, info.next, std::memory_order_acquire)) { - goto retry; - } - - if (!info.cur) { - return false; - } - - info.next = info.cur->next.load(std::memory_order_relaxed); - if (info.next.mark() != 0) { - // Node *cur is marked for deletion -> update the link and retire the element - - // (7) - this acquire-load synchronizes-with the release-CAS (8, 9, 10, 12, 15) - info.next = info.cur->next.load(std::memory_order_acquire).get(); - - // Try to splice out node - marked_ptr expected = info.cur.get(); - // (8) - this release-CAS synchronizes with the acquire-load (1, 2, 3, 4, 5, 6, 7, 13) - // and the acquire-CAS (11, 14) - // it is the head of a potential release sequence containing (11, 14) - if (!info.prev->compare_exchange_weak( - expected, info.next, std::memory_order_release, std::memory_order_relaxed)) { - backoff(); - goto retry; - } - info.cur.reclaim(); - } else { - if (info.prev->load(std::memory_order_relaxed) != info.cur.get()) { - goto retry; // cur might be cut from the hash_map. - } - - const auto& data = info.cur->data; - if (data.greater_or_equal(hash, key)) { - return data.value.first == key; - } - - info.prev = &info.cur->next; - std::swap(info.save, info.cur); - } - } -} - -template -bool harris_michael_hash_map::contains(const Key& key) { - auto h = hash{}(key); - auto bucket = map_to_bucket{}(h, num_buckets); - find_info info{&buckets[bucket]}; - backoff backoff; - return find(h, key, bucket, info, backoff); -} - -template -auto harris_michael_hash_map::find(const Key& key) -> iterator { - auto h = hash{}(key); - auto bucket = map_to_bucket{}(h, num_buckets); - find_info info{&buckets[bucket]}; - backoff backoff; - if (find(h, key, bucket, info, backoff)) { - return iterator(this, bucket, std::move(info)); - } - return end(); -} - -template -template -bool harris_michael_hash_map::emplace(Args&&... args) { - auto result = emplace_or_get(std::forward(args)...); - return result.second; -} - -template -template -auto harris_michael_hash_map::get_or_emplace(Key key, Args&&... args) - -> std::pair { - return do_get_or_emplace_lazy(std::move(key), [&args...](hash_t hash, Key key) { - return new node(hash, - std::piecewise_construct, - std::forward_as_tuple(std::move(key)), - std::forward_as_tuple(std::forward(args)...)); - }); -} - -template -template -auto harris_michael_hash_map::get_or_emplace_lazy(Key key, Factory value_factory) - -> std::pair { - return do_get_or_emplace_lazy( - std::move(key), [&value_factory](hash_t hash, Key key) { return new node(hash, std::move(key), value_factory()); }); -} - -template -template -auto harris_michael_hash_map::do_get_or_emplace_lazy(Key key, Factory node_factory) - -> std::pair { - node* n = nullptr; - auto h = hash{}(key); - auto bucket = map_to_bucket{}(h, num_buckets); - - const Key* pkey = &key; - find_info info{&buckets[bucket]}; - backoff backoff; - for (;;) { - if (find(h, *pkey, bucket, info, backoff)) { - delete n; - return {iterator(this, bucket, std::move(info)), false}; - } - if (n == nullptr) { - n = node_factory(h, std::move(key)); // NOLINT (use-after-move) - pkey = &n->data.value.first; - } - - // Try to install new node - marked_ptr cur = info.cur.get(); - info.cur.reset(); - info.cur = guard_ptr(n); - n->next.store(cur, std::memory_order_relaxed); - - // (9) - this release-CAS synchronizes with the acquire-load (1, 2, 3, 4, 5, 6, 7, 13) - // and the acquire-CAS (11, 14) - // it is the head of a potential release sequence containing (11, 14) - if (info.prev->compare_exchange_weak(cur, n, std::memory_order_release, std::memory_order_relaxed)) { - return {iterator(this, bucket, std::move(info)), true}; - } - - backoff(); - } -} - -template -template -auto harris_michael_hash_map::emplace_or_get(Args&&... args) -> std::pair { - node* n = new node(construct_without_hash{}, std::forward(args)...); - - auto h = n->data.get_hash(); - auto bucket = map_to_bucket{}(h, num_buckets); - - find_info info{&buckets[bucket]}; - backoff backoff; - for (;;) { - if (find(h, n->data.value.first, bucket, info, backoff)) { - delete n; - return {iterator(this, bucket, std::move(info)), false}; - } - // Try to install new node - marked_ptr expected = info.cur.get(); - n->next.store(expected, std::memory_order_relaxed); - guard_ptr new_guard(n); - - // (10) - this release-CAS synchronizes with the acquire-load (1, 2, 3, 4, 5, 6, 7, 13) - // and the acquire-CAS (11, 14) - // it is the head of a potential release sequence containing (11, 14) - if (info.prev->compare_exchange_weak(expected, n, std::memory_order_release, std::memory_order_relaxed)) { - info.cur = std::move(new_guard); - return {iterator(this, bucket, std::move(info)), true}; - } - - backoff(); - } -} - -template -bool harris_michael_hash_map::erase(const Key& key) { - auto h = hash{}(key); - auto bucket = map_to_bucket{}(h, num_buckets); - backoff backoff; - find_info info{&buckets[bucket]}; - // Find node in hash_map with matching key and mark it for erasure. - do { - if (!find(h, key, bucket, info, backoff)) { - return false; // No such node in the hash_map - } - // (11) - this acquire-CAS synchronizes with the release-CAS (8, 9, 10, 12, 15) - // and is part of a release sequence headed by those operations - } while (!info.cur->next.compare_exchange_weak( - info.next, marked_ptr(info.next.get(), 1), std::memory_order_acquire, std::memory_order_relaxed)); - - assert(info.next.mark() == 0); - assert(info.cur.mark() == 0); - - // Try to splice out node - marked_ptr expected = info.cur; - // (12) - this release-CAS synchronizes with the acquire-load (1, 2, 3, 4, 5, 6, 7, 13) - // and the acquire-CAS (11, 14) - // it is the head of a potential release sequence containing (11, 14) - if (info.prev->compare_exchange_weak( - expected, info.next.get(), std::memory_order_release, std::memory_order_relaxed)) { - info.cur.reclaim(); - } else { - // Another thread interfered -> rewalk the bucket's list to ensure - // reclamation of marked node before returning. - find(h, key, bucket, info, backoff); - } - - return true; -} - -template -auto harris_michael_hash_map::erase(iterator pos) -> iterator { - backoff backoff; - // (13) - this acquire-load synchronizes-with the release-CAS (8, 9, 10, 12, 15) - auto next = pos.info.cur->next.load(std::memory_order_acquire); - while (next.mark() == 0) { - // (14) - this acquire-CAS synchronizes with the release-CAS (8, 9, 10, 12, 15) - // and is part of a release sequence headed by those operations - if (pos.info.cur->next.compare_exchange_weak(next, marked_ptr(next.get(), 1), std::memory_order_acquire)) { - break; - } - - backoff(); - } - - guard_ptr next_guard(next.get()); - assert(pos.info.cur.mark() == 0); - - // Try to splice out node - marked_ptr expected = pos.info.cur; - // (15) - this release-CAS synchronizes with the acquire-load (1, 2, 3, 4, 5, 6, 7, 13) - // and the acquire-CAS (11, 14) - // it is the head of a potential release sequence containing (11, 14) - if (pos.info.prev->compare_exchange_weak( - expected, next_guard, std::memory_order_release, std::memory_order_relaxed)) { - pos.info.cur.reclaim(); - pos.info.cur = std::move(next_guard); - } else { - next_guard.reset(); - Key key = pos.info.cur->data.value.first; - hash_t h = pos.info.cur->data.get_hash(); - - // Another thread interfered -> rewalk the list to ensure reclamation of marked node before returning. - find(h, key, pos.bucket, pos.info, backoff); - } - - if (!pos.info.cur) { - pos.move_to_next_bucket(); - } - - return pos; -} - -template -auto harris_michael_hash_map::operator[](const Key& key) -> accessor { - auto result = get_or_emplace_lazy(key, []() { return Value{}; }); - return accessor(std::move(result.first.info.cur)); -} - -template -auto harris_michael_hash_map::begin() -> iterator { - return iterator(this, 0); -} - -template -auto harris_michael_hash_map::end() -> iterator { - return iterator(this); -} -} // namespace xenium - -#endif +// +// Copyright (c) 2018-2020 Manuel Pöter. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +#ifndef XENIUM_HARRIS_MICHAEL_HASH_MAP_HPP +#define XENIUM_HARRIS_MICHAEL_HASH_MAP_HPP + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace xenium { + +namespace policy { + /** + * @brief Policy to configure the number of buckets in `harris_michael_hash_map`. + * @tparam Value + */ + template + struct buckets; + + /** + * @brief Policy to configure the function that maps the hash value to a bucket + * in `harris_michael_hash_map`. + * + * This can be used to apply other multiplicative methods like fibonacci hashing. + * + * @tparam T + */ + template + struct map_to_bucket; + + /** + * @brief Policy to configure whether the hash value should be stored and used + * during lookup operations in `harris_michael_hash_map`. + * + * This can improve performance for complex types with expensive compare operations + * like strings. + * + * @tparam T + */ + template + struct memoize_hash; +} // namespace policy + +/** + * @brief A generic lock-free hash-map. + * + * This hash-map consists of a fixed number of buckets were each bucket is essentially + * a `harris_michael_list_based_set` instance. The number of buckets is fixed, so the + * hash-map does not support dynamic resizing. + * + * This hash-map is less efficient than many other available concurrent hash-maps, but it is + * lock-free and fully generic, i.e., it supports arbitrary types for `Key` and `Value`. + * + * This data structure is based on the solution proposed by Michael \[[Mic02](index.html#ref-michael-2002)\] + * which builds upon the original proposal by Harris \[[Har01](index.html#ref-harris-2001)\]. + * + * Supported policies: + * * `xenium::policy::reclaimer`
+ * Defines the reclamation scheme to be used for internal nodes. (**required**) + * * `xenium::policy::hash`
+ * Defines the hash function. (*optional*; defaults to `xenium::hash`) + * * `xenium::policy::map_to_bucket`
+ * Defines the function that is used to map the calculated hash to a bucket. + * (*optional*; defaults to `xenium::utils::modulo`) + * * `xenium::policy::backoff`
+ * Defines the backoff strategy. (*optional*; defaults to `xenium::no_backoff`) + * * `xenium::policy::buckets`
+ * Defines the number of buckets. (*optional*; defaults to 512) + * * `xenium::policy::memoize_hash`
+ * Defines whether the hash should be stored and used during lookup operations. + * (*optional*; defaults to false for scalar `Key` types; otherwise true) + * + * @tparam Key + * @tparam Value + * @tparam Policies list of policies to customize the behaviour + */ +template +class harris_michael_hash_map { +public: + using value_type = std::pair; + using reclaimer = parameter::type_param_t; + using hash = parameter::type_param_t, Policies...>; + using map_to_bucket = parameter::type_param_t, Policies...>; + using backoff = parameter::type_param_t; + static constexpr std::size_t num_buckets = + parameter::value_param_t::value; + static constexpr bool memoize_hash = + parameter::value_param_t::value, Policies...>::value; + + template + using with = harris_michael_hash_map; + + static_assert(parameter::is_set::value, "reclaimer policy must be specified"); + + class iterator; + class accessor; + + harris_michael_hash_map() = default; + ~harris_michael_hash_map(); + + /** + * @brief Inserts a new element into the container if the container doesn't already contain an + * element with an equivalent key. The element is constructed in-place with the given `args`. + * + * The element is always constructed. If there already is an element with the key in the container, + * the newly constructed element will be destroyed immediately. + * + * No iterators or references are invalidated. + * + * Progress guarantees: lock-free + * + * @param args arguments to forward to the constructor of the element + * @return `true` if an element was inserted, otherwise `false` + */ + template + bool emplace(Args&&... args); + + /** + * @brief Inserts a new element into the container if the container doesn't already contain an + * element with an equivalent key. The element is constructed in-place with the given `args`. + * + * The element is always constructed. If there already is an element with the key in the container, + * the newly constructed element will be destroyed immediately. + * + * No iterators or references are invalidated. + * + * Progress guarantees: lock-free + * + * @param args arguments to forward to the constructor of the element + * @return a pair consisting of an iterator to the inserted element, or the already-existing element + * if no insertion happened, and a bool denoting whether the insertion took place; + * `true` if an element was inserted, otherwise `false` + */ + template + std::pair emplace_or_get(Args&&... args); + + /** + * @brief Inserts a new element into the container if the container doesn't already contain an + * element with an equivalent key. The element is constructed as `value_type(std::piecewise_construct, + * std::forward_as_tuple(k), std::forward_as_tuple(std::forward(args)...))`. + * + * The element may be constructed even if there already is an element with the key in the container, + * in which case the newly constructed element will be destroyed immediately. + * + * No iterators or references are invalidated. + * Progress guarantees: lock-free + * + * @param key the key of element to be inserted. + * @param args arguments to forward to the constructor of the element + * @return a pair consisting of an iterator to the inserted element, or the already-existing element + * if no insertion happened, and a bool denoting whether the insertion took place; + * `true` if an element was inserted, otherwise `false` + */ + template + std::pair get_or_emplace(Key key, Args&&... args); + + /** + * @brief Inserts a new element into the container if the container doesn't already contain an + * element with an equivalent key. The value for the newly constructed element is created by + * calling `value_factory`. + * + * The element may be constructed even if there already is an element with the key in the container, + * in which case the newly constructed element will be destroyed immediately. + * + * No iterators or references are invalidated. + * Progress guarantees: lock-free + * + * @tparam Func + * @param key the key of element to be inserted. + * @param factory a functor that is used to create the `Value` instance when constructing + * the new element to be inserted. + * @return a pair consisting of an iterator to the inserted element, or the already-existing element + * if no insertion happened, and a bool denoting whether the insertion took place; + * `true` if an element was inserted, otherwise `false` + */ + template + std::pair get_or_emplace_lazy(Key key, Factory factory); + + /** + * @brief Removes the element with the key equivalent to key (if one exists). + * + * No iterators or references are invalidated. + * + * Progress guarantees: lock-free + * + * @param key key of the element to remove + * @return `true` if an element was removed, otherwise `false` + */ + bool erase(const Key& key); + + /** + * @brief Removes the specified element from the container. + * + * No iterators or references are invalidated. + * + * Progress guarantees: lock-free + * + * @param pos the iterator identifying the element to remove + * @return iterator following the last removed element + */ + iterator erase(iterator pos); + + /** + * @brief Finds an element with key equivalent to key. + * + * Progress guarantees: lock-free + * + * @param key key of the element to search for + * @return iterator to an element with key equivalent to key if such element is found, + * otherwise past-the-end iterator + */ + iterator find(const Key& key); + + /** + * @brief Checks if there is an element with key equivalent to key in the container. + * + * Progress guarantees: lock-free + * + * @param key key of the element to search for + * @return `true` if there is such an element, otherwise `false` + */ + bool contains(const Key& key); + + /** + * @brief + * + * The `accessor` + * @param key + * @return an accessor to the element's value + */ + accessor operator[](const Key& key); + + /** + * @brief Returns an iterator to the first element of the container. + * @return iterator to the first element + */ + iterator begin(); + + /** + * @brief Returns an iterator to the element following the last element of the container. + * + * This element acts as a placeholder; attempting to access it results in undefined behavior. + * @return iterator to the element following the last element. + */ + iterator end(); + +private: + struct node; + using hash_t = std::size_t; + using concurrent_ptr = typename reclaimer::template concurrent_ptr; + using marked_ptr = typename concurrent_ptr::marked_ptr; + using guard_ptr = typename concurrent_ptr::guard_ptr; + + template + std::pair do_get_or_emplace_lazy(Key key, Factory node_factory); + + struct construct_without_hash {}; + + struct data_without_hash { + value_type value; + template + explicit data_without_hash(hash_t /*hash*/, Args&&... args) : value(std::forward(args)...) {} + template + explicit data_without_hash(construct_without_hash, Args&&... args) : value(std::forward(args)...) {} + [[nodiscard]] hash_t get_hash() const { return hash{}(value.first); } + [[nodiscard]] bool greater_or_equal(hash_t /*h*/, const Key& key) const { return value.first >= key; } + }; + + struct data_with_hash { + hash_t hash; + value_type value; + template + explicit data_with_hash(hash_t hash, Args&&... args) : hash(hash), value(std::forward(args)...) {} + template + explicit data_with_hash(construct_without_hash, Args&&... args) : value(std::forward(args)...) { + hash = harris_michael_hash_map::hash{}(value.first); + } + [[nodiscard]] hash_t get_hash() const { return hash; } + [[nodiscard]] bool greater_or_equal(hash_t h, const Key& key) const { return hash >= h && value.first >= key; } + }; + + using data_t = std::conditional_t; + + struct node : reclaimer::template enable_concurrent_ptr { + data_t data; + concurrent_ptr next; + template + explicit node(Args&&... args) : data(std::forward(args)...), next() {} + }; + + struct find_info { + concurrent_ptr* prev; + marked_ptr next{}; + guard_ptr cur{}; + guard_ptr save{}; + }; + + bool find(hash_t hash, const Key& key, std::size_t bucket, find_info& info, backoff& backoff); + + concurrent_ptr buckets[num_buckets]; +}; + +/** + * @brief A ForwardIterator to safely iterate the hash-map. + * + * Iterators are not invalidated by concurrent insert/erase operations. However, conflicting erase + * operations can have a negative impact on the performance when advancing the iterator, because it + * may be necessary to rescan the bucket's list to find the next element. + * + * *Note:* This iterator class does *not* provide multi-pass guarantee as `a == b` does not imply `++a == ++b`. + * + * *Note:* Each iterator internally holds two `guard_ptr` instances. This has to be considered when using + * a reclamation scheme that requires per-instance resources like `hazard_pointer` or `hazard_eras`. + * It is therefore highly recommended to use prefix increments wherever possible. + */ +template +class harris_michael_hash_map::iterator { +public: + using iterator_category = std::forward_iterator_tag; + using value_type = harris_michael_hash_map::value_type; + using difference_type = std::ptrdiff_t; + using pointer = value_type*; + using reference = value_type&; + + iterator(iterator&&) = default; + iterator(const iterator&) = default; + + iterator& operator=(iterator&&) = default; + iterator& operator=(const iterator&) = default; + + iterator& operator++() { + assert(info.cur.get() != nullptr); + auto next = info.cur->next.load(std::memory_order_relaxed); + guard_ptr tmp_guard; + // (1) - this acquire-load synchronizes-with the release-CAS (8, 9, 10, 12, 15) + if (next.mark() == 0 && tmp_guard.acquire_if_equal(info.cur->next, next, std::memory_order_acquire)) { + info.prev = &info.cur->next; + info.save = std::move(info.cur); + info.cur = std::move(tmp_guard); + } else { + // cur is marked for removal + // -> use find to remove it and get to the next node with a key >= cur->key + // Note: we have to copy key here! + Key key = info.cur->data.value.first; + hash_t h = info.cur->data.get_hash(); + backoff backoff; + map->find(h, key, bucket, info, backoff); + } + assert(info.prev == &map->buckets[bucket] || info.cur.get() == nullptr || + (info.save.get() != nullptr && &info.save->next == info.prev)); + + if (!info.cur) { + move_to_next_bucket(); + } + + return *this; + } + iterator operator++(int) { + iterator retval = *this; + ++(*this); + return retval; + } + bool operator==(const iterator& other) const { return info.cur.get() == other.info.cur.get(); } + bool operator!=(const iterator& other) const { return !(*this == other); } + reference operator*() const noexcept { return info.cur->data.value; } + pointer operator->() const noexcept { return &info.cur->data.value; } + + void reset() { + bucket = num_buckets; + info.prev = nullptr; + info.cur.reset(); + info.save.reset(); + } + +private: + friend harris_michael_hash_map; + + explicit iterator(harris_michael_hash_map* map) : map(map), bucket(num_buckets) {} + + explicit iterator(harris_michael_hash_map* map, std::size_t bucket) : map(map), bucket(bucket) { + info.prev = &map->buckets[bucket]; + // (2) - this acquire-load synchronizes-with the release-CAS (8, 9, 10, 12, 15) + info.cur.acquire(*info.prev, std::memory_order_acquire); + + if (!info.cur) { + move_to_next_bucket(); + } + } + + explicit iterator(harris_michael_hash_map* map, std::size_t bucket, find_info&& info) : + map(map), + bucket(bucket), + info(std::move(info)) {} + + void move_to_next_bucket() { + info.save.reset(); + while (!info.cur && bucket < num_buckets - 1) { + ++bucket; + info.prev = &map->buckets[bucket]; + // (3) - this acquire-load synchronizes-with the release-CAS (8, 9, 10, 12, 15) + info.cur.acquire(*info.prev, std::memory_order_acquire); + } + } + + harris_michael_hash_map* map; + std::size_t bucket; + find_info info; +}; + +/** + * @brief An accessor to safely access the value of an element. + * + * It is used as result type of `operator[]` to provide a similar behaviour as the + * STL `map`/`unordered_map` classes. However, the accessor has to be dereferenced to + * access the underlying value (`operator*`). + * + * Note: an `accessor` instance internally holds a `guard_ptr` to the value's element. + */ +template +class harris_michael_hash_map::accessor { +public: + Value* operator->() const noexcept { return &guard->data.value.second; } + Value& operator*() const noexcept { return guard->data.value.second; } + void reset() { guard.reset(); } + +private: + explicit accessor(guard_ptr&& guard) : guard(std::move(guard)) {} + guard_ptr guard; + friend harris_michael_hash_map; +}; + +template +harris_michael_hash_map::~harris_michael_hash_map() { + for (std::size_t i = 0; i < num_buckets; ++i) { + // (4) - this acquire-load synchronizes-with the release-CAS (8, 9, 10, 12, 15) + auto p = buckets[i].load(std::memory_order_acquire); + while (p) { + // (5) - this acquire-load synchronizes-with the release-CAS (8, 9, 10, 12, 15) + auto next = p->next.load(std::memory_order_acquire); + delete p.get(); + p = next; + } + } +} + +template +bool harris_michael_hash_map::find(hash_t hash, + const Key& key, + std::size_t bucket, + find_info& info, + backoff& backoff) { + auto& head = buckets[bucket]; + assert((info.save == nullptr && info.prev == &head) || &info.save->next == info.prev); + concurrent_ptr* start = info.prev; + guard_ptr start_guard = info.save; // we have to keep a guard_ptr to prevent start's node from getting reclaimed. +retry: + info.prev = start; + info.save = start_guard; + info.next = info.prev->load(std::memory_order_relaxed); + if (info.next.mark() != 0) { + // our start node is marked for removal -> we have to restart from head + start = &head; + start_guard.reset(); + goto retry; + } + + for (;;) { + // (6) - this acquire-load synchronizes-with the release-CAS (8, 9, 10, 12, 15) + if (!info.cur.acquire_if_equal(*info.prev, info.next, std::memory_order_acquire)) { + goto retry; + } + + if (!info.cur) { + return false; + } + + info.next = info.cur->next.load(std::memory_order_relaxed); + if (info.next.mark() != 0) { + // Node *cur is marked for deletion -> update the link and retire the element + + // (7) - this acquire-load synchronizes-with the release-CAS (8, 9, 10, 12, 15) + info.next = info.cur->next.load(std::memory_order_acquire).get(); + + // Try to splice out node + marked_ptr expected = info.cur.get(); + // (8) - this release-CAS synchronizes with the acquire-load (1, 2, 3, 4, 5, 6, 7, 13) + // and the acquire-CAS (11, 14) + // it is the head of a potential release sequence containing (11, 14) + if (!info.prev->compare_exchange_weak( + expected, info.next, std::memory_order_release, std::memory_order_relaxed)) { + backoff(); + goto retry; + } + info.cur.reclaim(); + } else { + if (info.prev->load(std::memory_order_relaxed) != info.cur.get()) { + goto retry; // cur might be cut from the hash_map. + } + + const auto& data = info.cur->data; + if (data.greater_or_equal(hash, key)) { + return data.value.first == key; + } + + info.prev = &info.cur->next; + std::swap(info.save, info.cur); + } + } +} + +template +bool harris_michael_hash_map::contains(const Key& key) { + auto h = hash{}(key); + auto bucket = map_to_bucket{}(h, num_buckets); + find_info info{&buckets[bucket]}; + backoff backoff; + return find(h, key, bucket, info, backoff); +} + +template +auto harris_michael_hash_map::find(const Key& key) -> iterator { + auto h = hash{}(key); + auto bucket = map_to_bucket{}(h, num_buckets); + find_info info{&buckets[bucket]}; + backoff backoff; + if (find(h, key, bucket, info, backoff)) { + return iterator(this, bucket, std::move(info)); + } + return end(); +} + +template +template +bool harris_michael_hash_map::emplace(Args&&... args) { + auto result = emplace_or_get(std::forward(args)...); + return result.second; +} + +template +template +auto harris_michael_hash_map::get_or_emplace(Key key, Args&&... args) + -> std::pair { + return do_get_or_emplace_lazy(std::move(key), [&args...](hash_t hash, Key key) { + return new node(hash, + std::piecewise_construct, + std::forward_as_tuple(std::move(key)), + std::forward_as_tuple(std::forward(args)...)); + }); +} + +template +template +auto harris_michael_hash_map::get_or_emplace_lazy(Key key, Factory value_factory) + -> std::pair { + return do_get_or_emplace_lazy( + std::move(key), [&value_factory](hash_t hash, Key key) { return new node(hash, std::move(key), value_factory()); }); +} + +template +template +auto harris_michael_hash_map::do_get_or_emplace_lazy(Key key, Factory node_factory) + -> std::pair { + node* n = nullptr; + auto h = hash{}(key); + auto bucket = map_to_bucket{}(h, num_buckets); + + const Key* pkey = &key; + find_info info{&buckets[bucket]}; + backoff backoff; + for (;;) { + if (find(h, *pkey, bucket, info, backoff)) { + delete n; + return {iterator(this, bucket, std::move(info)), false}; + } + if (n == nullptr) { + n = node_factory(h, std::move(key)); // NOLINT (use-after-move) + pkey = &n->data.value.first; + } + + // Try to install new node + marked_ptr cur = info.cur.get(); + info.cur.reset(); + info.cur = guard_ptr(n); + n->next.store(cur, std::memory_order_relaxed); + + // (9) - this release-CAS synchronizes with the acquire-load (1, 2, 3, 4, 5, 6, 7, 13) + // and the acquire-CAS (11, 14) + // it is the head of a potential release sequence containing (11, 14) + if (info.prev->compare_exchange_weak(cur, n, std::memory_order_release, std::memory_order_relaxed)) { + return {iterator(this, bucket, std::move(info)), true}; + } + + backoff(); + } +} + +template +template +auto harris_michael_hash_map::emplace_or_get(Args&&... args) -> std::pair { + node* n = new node(construct_without_hash{}, std::forward(args)...); + + auto h = n->data.get_hash(); + auto bucket = map_to_bucket{}(h, num_buckets); + + find_info info{&buckets[bucket]}; + backoff backoff; + for (;;) { + if (find(h, n->data.value.first, bucket, info, backoff)) { + delete n; + return {iterator(this, bucket, std::move(info)), false}; + } + // Try to install new node + marked_ptr expected = info.cur.get(); + n->next.store(expected, std::memory_order_relaxed); + guard_ptr new_guard(n); + + // (10) - this release-CAS synchronizes with the acquire-load (1, 2, 3, 4, 5, 6, 7, 13) + // and the acquire-CAS (11, 14) + // it is the head of a potential release sequence containing (11, 14) + if (info.prev->compare_exchange_weak(expected, n, std::memory_order_release, std::memory_order_relaxed)) { + info.cur = std::move(new_guard); + return {iterator(this, bucket, std::move(info)), true}; + } + + backoff(); + } +} + +template +bool harris_michael_hash_map::erase(const Key& key) { + auto h = hash{}(key); + auto bucket = map_to_bucket{}(h, num_buckets); + backoff backoff; + find_info info{&buckets[bucket]}; + // Find node in hash_map with matching key and mark it for erasure. + do { + if (!find(h, key, bucket, info, backoff)) { + return false; // No such node in the hash_map + } + // (11) - this acquire-CAS synchronizes with the release-CAS (8, 9, 10, 12, 15) + // and is part of a release sequence headed by those operations + } while (!info.cur->next.compare_exchange_weak( + info.next, marked_ptr(info.next.get(), 1), std::memory_order_acquire, std::memory_order_relaxed)); + + assert(info.next.mark() == 0); + assert(info.cur.mark() == 0); + + // Try to splice out node + marked_ptr expected = info.cur; + // (12) - this release-CAS synchronizes with the acquire-load (1, 2, 3, 4, 5, 6, 7, 13) + // and the acquire-CAS (11, 14) + // it is the head of a potential release sequence containing (11, 14) + if (info.prev->compare_exchange_weak( + expected, info.next.get(), std::memory_order_release, std::memory_order_relaxed)) { + info.cur.reclaim(); + } else { + // Another thread interfered -> rewalk the bucket's list to ensure + // reclamation of marked node before returning. + find(h, key, bucket, info, backoff); + } + + return true; +} + +template +auto harris_michael_hash_map::erase(iterator pos) -> iterator { + backoff backoff; + // (13) - this acquire-load synchronizes-with the release-CAS (8, 9, 10, 12, 15) + auto next = pos.info.cur->next.load(std::memory_order_acquire); + while (next.mark() == 0) { + // (14) - this acquire-CAS synchronizes with the release-CAS (8, 9, 10, 12, 15) + // and is part of a release sequence headed by those operations + if (pos.info.cur->next.compare_exchange_weak(next, marked_ptr(next.get(), 1), std::memory_order_acquire)) { + break; + } + + backoff(); + } + + guard_ptr next_guard(next.get()); + assert(pos.info.cur.mark() == 0); + + // Try to splice out node + marked_ptr expected = pos.info.cur; + // (15) - this release-CAS synchronizes with the acquire-load (1, 2, 3, 4, 5, 6, 7, 13) + // and the acquire-CAS (11, 14) + // it is the head of a potential release sequence containing (11, 14) + if (pos.info.prev->compare_exchange_weak( + expected, next_guard, std::memory_order_release, std::memory_order_relaxed)) { + pos.info.cur.reclaim(); + pos.info.cur = std::move(next_guard); + } else { + next_guard.reset(); + Key key = pos.info.cur->data.value.first; + hash_t h = pos.info.cur->data.get_hash(); + + // Another thread interfered -> rewalk the list to ensure reclamation of marked node before returning. + find(h, key, pos.bucket, pos.info, backoff); + } + + if (!pos.info.cur) { + pos.move_to_next_bucket(); + } + + return pos; +} + +template +auto harris_michael_hash_map::operator[](const Key& key) -> accessor { + auto result = get_or_emplace_lazy(key, []() { return Value{}; }); + return accessor(std::move(result.first.info.cur)); +} + +template +auto harris_michael_hash_map::begin() -> iterator { + return iterator(this, 0); +} + +template +auto harris_michael_hash_map::end() -> iterator { + return iterator(this); +} +} // namespace xenium + +#endif diff --git a/xenium/harris_michael_list_based_set.hpp b/include/xenium/harris_michael_list_based_set.hpp similarity index 97% rename from xenium/harris_michael_list_based_set.hpp rename to include/xenium/harris_michael_list_based_set.hpp index 289b202..f3c7f89 100644 --- a/xenium/harris_michael_list_based_set.hpp +++ b/include/xenium/harris_michael_list_based_set.hpp @@ -1,493 +1,493 @@ -// -// Copyright (c) 2018-2020 Manuel Pöter. -// Licensed under the MIT License. See LICENSE file in the project root for full license information. -// - -#ifndef XENIUM_HARRIS_MICHAEL_LIST_BASED_SET_HPP -#define XENIUM_HARRIS_MICHAEL_LIST_BASED_SET_HPP - -#include -#include -#include -#include - -#include -#include - -namespace xenium { -/** - * @brief A lock-free container that contains a sorted set of unique objects of type `Key`. - * - * This container is implemented as a sorted singly linked list. All operations have - * a runtime complexity linear in the size of the list (in the absence of conflicting - * operations). - * - * This data structure is based on the solution proposed by Michael \[[Mic02](index.html#ref-michael-2002)\] - * which builds upon the original proposal by Harris \[[Har01](index.html#ref-harris-2001)\]. - * - * * Supported policies: - * * `xenium::policy::reclaimer`
- * Defines the reclamation scheme to be used for internal nodes. (**required**) - * * `xenium::policy::compare`
- * Defines the comparison function that is used to order the list. (*optional*; defaults to `std::less`) - * * `xenium::policy::backoff`
- * Defines the backoff strategy. (*optional*; defaults to `xenium::no_backoff`) - * - * @tparam Key type of the stored elements. - * @tparam Policies list of policies to customize the behaviour - */ -template -class harris_michael_list_based_set { -public: - using value_type = Key; - using reclaimer = parameter::type_param_t; - using backoff = parameter::type_param_t; - using compare = parameter::type_param_t, Policies...>; - - template - using with = harris_michael_list_based_set; - - static_assert(parameter::is_set::value, "reclaimer policy must be specified"); - - harris_michael_list_based_set() = default; - ~harris_michael_list_based_set(); - - class iterator; - - /** - * @brief Inserts a new element into the container if the container doesn't already contain an - * element with an equivalent key. The element is constructed in-place with the given `args`. - * - * The element is always constructed. If there already is an element with the key in the container, - * the newly constructed element will be destroyed immediately. - * - * No iterators or references are invalidated. - * - * Progress guarantees: lock-free - * - * @param args arguments to forward to the constructor of the element - * @return `true` if an element was inserted, otherwise `false` - */ - template - bool emplace(Args&&... args); - - /** - * @brief Inserts a new element into the container if the container doesn't already contain an - * element with an equivalent key. The element is constructed in-place with the given `args`. - * - * The element is always constructed. If there already is an element with the key in the container, - * the newly constructed element will be destroyed immediately. - * - * No iterators or references are invalidated. - * - * Progress guarantees: lock-free - * - * @param args arguments to forward to the constructor of the element - * @return a pair consisting of an iterator to the inserted element, or the already-existing element - * if no insertion happened, and a bool denoting whether the insertion took place; - * `true` if an element was inserted, otherwise `false` - */ - template - std::pair emplace_or_get(Args&&... args); - - /** - * @brief Removes the element with the key equivalent to key (if one exists). - * - * No iterators or references are invalidated. - * - * Progress guarantees: lock-free - * - * @param key key of the element to remove - * @return `true` if an element was removed, otherwise `false` - */ - bool erase(const Key& key); - - /** - * @brief Removes the specified element from the container. - * - * No iterators or references are invalidated. - * - * Progress guarantees: lock-free - * - * @param pos the iterator identifying the element to remove - * @return iterator following the last removed element - */ - iterator erase(iterator pos); - - /** - * @brief Finds an element with key equivalent to key. - * - * Progress guarantees: lock-free - * - * @param key key of the element to search for - * @return iterator to an element with key equivalent to key if such element is found, - * otherwise past-the-end iterator - */ - iterator find(const Key& key); - - /** - * @brief Checks if there is an element with key equivalent to key in the container. - * - * Progress guarantees: lock-free - * - * @param key key of the element to search for - * @return `true` if there is such an element, otherwise `false` - */ - bool contains(const Key& key); - - /** - * @brief Returns an iterator to the first element of the container. - * @return iterator to the first element - */ - iterator begin(); - - /** - * @brief Returns an iterator to the element following the last element of the container. - * - * This element acts as a placeholder; attempting to access it results in undefined behavior. - * @return iterator to the element following the last element. - */ - iterator end(); - -private: - struct node; - - using concurrent_ptr = typename reclaimer::template concurrent_ptr; - using marked_ptr = typename concurrent_ptr::marked_ptr; - using guard_ptr = typename concurrent_ptr::guard_ptr; - - struct find_info { - concurrent_ptr* prev; - marked_ptr next{}; - guard_ptr cur{}; - guard_ptr save{}; - }; - bool find(const Key& key, find_info& info, backoff& backoff); - - concurrent_ptr head; -}; - -/** - * @brief A ForwardIterator to safely iterate the list. - * - * Iterators are not invalidated by concurrent insert/erase operations. However, conflicting erase - * operations can have a negative impact on the performance when advancing the iterator, because it - * may be necessary to rescan the list to find the next element. - * - * *Note:* This iterator class does *not* provide multi-pass guarantee as `a == b` does not imply `++a == ++b`. - * - * *Note:* Each iterator internally holds two `guard_ptr` instances. This has to be considered when using - * a reclamation scheme that requires per-instance resources like `hazard_pointer` or `hazard_eras`. - * It is therefore highly recommended to use prefix increments wherever possible. - */ -template -class harris_michael_list_based_set::iterator { -public: - using iterator_category = std::forward_iterator_tag; - using value_type = Key; - using difference_type = std::ptrdiff_t; - using pointer = const Key*; - using reference = const Key&; - - iterator(iterator&&) = default; - iterator(const iterator&) = default; - - iterator& operator=(iterator&&) = default; - iterator& operator=(const iterator&) = default; - - /** - * @brief Moves the iterator to the next element. - * In the absence of conflicting operations, this operation has constant runtime complexity. - * However, in case of conflicting erase operations we might have to rescan the list to help - * remove the node and find the next element. - * - * Progress guarantess: lock-free - */ - iterator& operator++(); - iterator operator++(int); - - bool operator==(const iterator& other) const { return info.cur.get() == other.info.cur.get(); } - bool operator!=(const iterator& other) const { return !(*this == other); } - reference operator*() const noexcept { return info.cur->key; } - pointer operator->() const noexcept { return &info.cur->key; } - - /** - * @brief Resets the iterator; this is equivalent to assigning `end()` to it. - * - * This operation can be handy in situations where an iterator is no longer needed and you want - * to ensure that the internal `guard_ptr` instances are reset. - */ - void reset() { - info.cur.reset(); - info.save.reset(); - } - -private: - friend harris_michael_list_based_set; - - explicit iterator(harris_michael_list_based_set& list, concurrent_ptr* start) : list(&list) { - info.prev = start; - if (start) { - // (2) - this acquire-load synchronizes-with the release-CAS (7, 8, 10, 13) - info.cur.acquire(*start, std::memory_order_acquire); - } - } - - explicit iterator(harris_michael_list_based_set& list, find_info&& info) : list(&list), info(std::move(info)) {} - - harris_michael_list_based_set* list; - find_info info; -}; - -template -struct harris_michael_list_based_set::node : reclaimer::template enable_concurrent_ptr { - const Key key; - concurrent_ptr next; - template - explicit node(Args&&... args) : key(std::forward(args)...), next() {} -}; - -template -auto harris_michael_list_based_set::iterator::operator++() -> iterator& { - assert(info.cur.get() != nullptr); - auto next = info.cur->next.load(std::memory_order_relaxed); - guard_ptr tmp_guard; - // (1) - this acquire-load synchronizes-with the release-CAS (7, 8, 10, 13) - if (next.mark() == 0 && tmp_guard.acquire_if_equal(info.cur->next, next, std::memory_order_acquire)) { - info.prev = &info.cur->next; - info.save = std::move(info.cur); - info.cur = std::move(tmp_guard); - } else { - // cur is marked for removal - // -> use find to remove it and get to the next node with a compare(key, cur->key) == false - auto key = info.cur->key; - backoff backoff; - list->find(key, info, backoff); - } - assert(info.prev == &list->head || info.cur.get() == nullptr || - (info.save.get() != nullptr && &info.save->next == info.prev)); - return *this; -} - -template -auto harris_michael_list_based_set::iterator::operator++(int) -> iterator { - iterator retval = *this; - ++(*this); - return retval; -} - -template -harris_michael_list_based_set::~harris_michael_list_based_set() { - // delete all remaining nodes - // (3) - this acquire-load synchronizes-with the release-CAS (7, 8, 10, 13) - auto p = head.load(std::memory_order_acquire); - while (p) { - // (4) - this acquire-load synchronizes-with the release-CAS (7, 8, 10, 13) - auto next = p->next.load(std::memory_order_acquire); - delete p.get(); - p = next; - } -} - -template -bool harris_michael_list_based_set::find(const Key& key, find_info& info, backoff& backoff) { - assert((info.save == nullptr && info.prev == &head) || &info.save->next == info.prev); - concurrent_ptr* start = info.prev; - guard_ptr start_guard = info.save; // we have to keep a guard_ptr to prevent start's node from getting reclaimed. -retry: - info.prev = start; - info.save = start_guard; - info.next = info.prev->load(std::memory_order_relaxed); - if (info.next.mark() != 0) { - // our start node is marked for removal -> we have to restart from head - start = &head; - start_guard.reset(); - goto retry; - } - - for (;;) { - // (5) - this acquire-load synchronizes-with the release-CAS (7, 8, 10, 13) - if (!info.cur.acquire_if_equal(*info.prev, info.next, std::memory_order_acquire)) { - goto retry; - } - - if (!info.cur) { - return false; - } - - info.next = info.cur->next.load(std::memory_order_relaxed); - if (info.next.mark() != 0) { - // Node *cur is marked for deletion -> update the link and retire the element - - // (6) - this acquire-load synchronizes-with the release-CAS (7, 8, 10, 13) - info.next = info.cur->next.load(std::memory_order_acquire).get(); - - // Try to splice out node - marked_ptr expected = info.cur.get(); - // (7) - this release-CAS synchronizes with the acquire-load (1, 2, 3, 4, 5, 6, 11) - // and the require-CAS (9, 12) - // it is the head of a potential release sequence containing (9, 12) - if (!info.prev->compare_exchange_weak( - expected, info.next, std::memory_order_release, std::memory_order_relaxed)) { - backoff(); - goto retry; - } - info.cur.reclaim(); - } else { - if (info.prev->load(std::memory_order_relaxed) != info.cur.get()) { - goto retry; // cur might be cut from list. - } - - const Key& ckey = info.cur->key; - compare compare; - if (!compare(ckey, key)) { - return !compare(key, ckey); - } - - info.prev = &info.cur->next; - std::swap(info.save, info.cur); - } - } -} - -template -bool harris_michael_list_based_set::contains(const Key& key) { - find_info info{&head}; - backoff backoff; - return find(key, info, backoff); -} - -template -auto harris_michael_list_based_set::find(const Key& key) -> iterator { - find_info info{&head}; - backoff backoff; - if (find(key, info, backoff)) { - return iterator(*this, std::move(info)); - } - return end(); -} - -template -template -bool harris_michael_list_based_set::emplace(Args&&... args) { - auto result = emplace_or_get(std::forward(args)...); - return result.second; -} - -template -template -auto harris_michael_list_based_set::emplace_or_get(Args&&... args) -> std::pair { - node* n = new node(std::forward(args)...); - find_info info{&head}; - backoff backoff; - for (;;) { - if (find(n->key, info, backoff)) { - delete n; - return {iterator(*this, std::move(info)), false}; - } - - // Try to install new node - marked_ptr expected = info.cur.get(); - n->next.store(expected, std::memory_order_relaxed); - guard_ptr new_guard(n); - - // (8) - this release-CAS synchronizes with the acquire-load (1, 2, 3, 4, 5, 6, 11) - // and the acquire-CAS (9, 12) - // it is the head of a potential release sequence containing (9, 12) - if (info.prev->compare_exchange_weak(expected, n, std::memory_order_release, std::memory_order_relaxed)) { - info.cur = std::move(new_guard); - return {iterator(*this, std::move(info)), true}; - } - - backoff(); - } -} - -template -bool harris_michael_list_based_set::erase(const Key& key) { - backoff backoff; - find_info info{&head}; - // Find node in list with matching key and mark it for reclamation. - for (;;) { - if (!find(key, info, backoff)) { - return false; // No such node in the list - } - - // (9) - this acquire-CAS synchronizes with the release-CAS (7, 8, 10, 13) - // and is part of a release sequence headed by those operations - if (info.cur->next.compare_exchange_weak( - info.next, marked_ptr(info.next.get(), 1), std::memory_order_acquire, std::memory_order_relaxed)) { - break; - } - - backoff(); - } - - assert(info.next.mark() == 0); - assert(info.cur.mark() == 0); - - // Try to splice out node - marked_ptr expected = info.cur; - // (10) - this release-CAS synchronizes with the acquire-load (1, 2, 3, 4, 5, 6, 11) - // and the acquire-CAS (9, 12) - // it is the head of a potential release sequence containing (9, 12) - if (info.prev->compare_exchange_weak(expected, info.next, std::memory_order_release, std::memory_order_relaxed)) { - info.cur.reclaim(); - } else { - // Another thread interfered -> rewalk the list to ensure reclamation of marked node before returning. - find(key, info, backoff); - } - - return true; -} - -template -auto harris_michael_list_based_set::erase(iterator pos) -> iterator { - backoff backoff; - // (11) - this acquire-load synchronizes-with the release-CAS (7, 8, 10, 13) - auto next = pos.info.cur->next.load(std::memory_order_acquire); - while (next.mark() == 0) { - // (12) - this acquire-CAS synchronizes-with the release-CAS (7, 8, 10, 13) - // and is part of a release sequence headed by those operations - if (pos.info.cur->next.compare_exchange_weak(next, marked_ptr(next.get(), 1), std::memory_order_acquire)) { - break; - } - - backoff(); - } - - guard_ptr next_guard(next.get()); - assert(pos.info.cur.mark() == 0); - - // Try to splice out node - marked_ptr expected = pos.info.cur; - // (13) - this release-CAS synchronizes with the acquire-load (1, 2, 3, 4, 5, 6, 11) - // and the acquire-CAS (9, 12) - // it is the head of a potential release sequence containing (9, 12) - if (pos.info.prev->compare_exchange_weak( - expected, next_guard, std::memory_order_release, std::memory_order_relaxed)) { - pos.info.cur.reclaim(); - pos.info.cur = std::move(next_guard); - } else { - next_guard.reset(); - Key key = pos.info.cur->key; - - // Another thread interfered -> rewalk the list to ensure reclamation of marked node before returning. - find(key, pos.info, backoff); - } - - return pos; -} - -template -auto harris_michael_list_based_set::begin() -> iterator { - return iterator(*this, &head); -} - -template -auto harris_michael_list_based_set::end() -> iterator { - return iterator(*this, nullptr); -} -} // namespace xenium - -#endif +// +// Copyright (c) 2018-2020 Manuel Pöter. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +#ifndef XENIUM_HARRIS_MICHAEL_LIST_BASED_SET_HPP +#define XENIUM_HARRIS_MICHAEL_LIST_BASED_SET_HPP + +#include +#include +#include +#include + +#include +#include + +namespace xenium { +/** + * @brief A lock-free container that contains a sorted set of unique objects of type `Key`. + * + * This container is implemented as a sorted singly linked list. All operations have + * a runtime complexity linear in the size of the list (in the absence of conflicting + * operations). + * + * This data structure is based on the solution proposed by Michael \[[Mic02](index.html#ref-michael-2002)\] + * which builds upon the original proposal by Harris \[[Har01](index.html#ref-harris-2001)\]. + * + * * Supported policies: + * * `xenium::policy::reclaimer`
+ * Defines the reclamation scheme to be used for internal nodes. (**required**) + * * `xenium::policy::compare`
+ * Defines the comparison function that is used to order the list. (*optional*; defaults to `std::less`) + * * `xenium::policy::backoff`
+ * Defines the backoff strategy. (*optional*; defaults to `xenium::no_backoff`) + * + * @tparam Key type of the stored elements. + * @tparam Policies list of policies to customize the behaviour + */ +template +class harris_michael_list_based_set { +public: + using value_type = Key; + using reclaimer = parameter::type_param_t; + using backoff = parameter::type_param_t; + using compare = parameter::type_param_t, Policies...>; + + template + using with = harris_michael_list_based_set; + + static_assert(parameter::is_set::value, "reclaimer policy must be specified"); + + harris_michael_list_based_set() = default; + ~harris_michael_list_based_set(); + + class iterator; + + /** + * @brief Inserts a new element into the container if the container doesn't already contain an + * element with an equivalent key. The element is constructed in-place with the given `args`. + * + * The element is always constructed. If there already is an element with the key in the container, + * the newly constructed element will be destroyed immediately. + * + * No iterators or references are invalidated. + * + * Progress guarantees: lock-free + * + * @param args arguments to forward to the constructor of the element + * @return `true` if an element was inserted, otherwise `false` + */ + template + bool emplace(Args&&... args); + + /** + * @brief Inserts a new element into the container if the container doesn't already contain an + * element with an equivalent key. The element is constructed in-place with the given `args`. + * + * The element is always constructed. If there already is an element with the key in the container, + * the newly constructed element will be destroyed immediately. + * + * No iterators or references are invalidated. + * + * Progress guarantees: lock-free + * + * @param args arguments to forward to the constructor of the element + * @return a pair consisting of an iterator to the inserted element, or the already-existing element + * if no insertion happened, and a bool denoting whether the insertion took place; + * `true` if an element was inserted, otherwise `false` + */ + template + std::pair emplace_or_get(Args&&... args); + + /** + * @brief Removes the element with the key equivalent to key (if one exists). + * + * No iterators or references are invalidated. + * + * Progress guarantees: lock-free + * + * @param key key of the element to remove + * @return `true` if an element was removed, otherwise `false` + */ + bool erase(const Key& key); + + /** + * @brief Removes the specified element from the container. + * + * No iterators or references are invalidated. + * + * Progress guarantees: lock-free + * + * @param pos the iterator identifying the element to remove + * @return iterator following the last removed element + */ + iterator erase(iterator pos); + + /** + * @brief Finds an element with key equivalent to key. + * + * Progress guarantees: lock-free + * + * @param key key of the element to search for + * @return iterator to an element with key equivalent to key if such element is found, + * otherwise past-the-end iterator + */ + iterator find(const Key& key); + + /** + * @brief Checks if there is an element with key equivalent to key in the container. + * + * Progress guarantees: lock-free + * + * @param key key of the element to search for + * @return `true` if there is such an element, otherwise `false` + */ + bool contains(const Key& key); + + /** + * @brief Returns an iterator to the first element of the container. + * @return iterator to the first element + */ + iterator begin(); + + /** + * @brief Returns an iterator to the element following the last element of the container. + * + * This element acts as a placeholder; attempting to access it results in undefined behavior. + * @return iterator to the element following the last element. + */ + iterator end(); + +private: + struct node; + + using concurrent_ptr = typename reclaimer::template concurrent_ptr; + using marked_ptr = typename concurrent_ptr::marked_ptr; + using guard_ptr = typename concurrent_ptr::guard_ptr; + + struct find_info { + concurrent_ptr* prev; + marked_ptr next{}; + guard_ptr cur{}; + guard_ptr save{}; + }; + bool find(const Key& key, find_info& info, backoff& backoff); + + concurrent_ptr head; +}; + +/** + * @brief A ForwardIterator to safely iterate the list. + * + * Iterators are not invalidated by concurrent insert/erase operations. However, conflicting erase + * operations can have a negative impact on the performance when advancing the iterator, because it + * may be necessary to rescan the list to find the next element. + * + * *Note:* This iterator class does *not* provide multi-pass guarantee as `a == b` does not imply `++a == ++b`. + * + * *Note:* Each iterator internally holds two `guard_ptr` instances. This has to be considered when using + * a reclamation scheme that requires per-instance resources like `hazard_pointer` or `hazard_eras`. + * It is therefore highly recommended to use prefix increments wherever possible. + */ +template +class harris_michael_list_based_set::iterator { +public: + using iterator_category = std::forward_iterator_tag; + using value_type = Key; + using difference_type = std::ptrdiff_t; + using pointer = const Key*; + using reference = const Key&; + + iterator(iterator&&) = default; + iterator(const iterator&) = default; + + iterator& operator=(iterator&&) = default; + iterator& operator=(const iterator&) = default; + + /** + * @brief Moves the iterator to the next element. + * In the absence of conflicting operations, this operation has constant runtime complexity. + * However, in case of conflicting erase operations we might have to rescan the list to help + * remove the node and find the next element. + * + * Progress guarantess: lock-free + */ + iterator& operator++(); + iterator operator++(int); + + bool operator==(const iterator& other) const { return info.cur.get() == other.info.cur.get(); } + bool operator!=(const iterator& other) const { return !(*this == other); } + reference operator*() const noexcept { return info.cur->key; } + pointer operator->() const noexcept { return &info.cur->key; } + + /** + * @brief Resets the iterator; this is equivalent to assigning `end()` to it. + * + * This operation can be handy in situations where an iterator is no longer needed and you want + * to ensure that the internal `guard_ptr` instances are reset. + */ + void reset() { + info.cur.reset(); + info.save.reset(); + } + +private: + friend harris_michael_list_based_set; + + explicit iterator(harris_michael_list_based_set& list, concurrent_ptr* start) : list(&list) { + info.prev = start; + if (start) { + // (2) - this acquire-load synchronizes-with the release-CAS (7, 8, 10, 13) + info.cur.acquire(*start, std::memory_order_acquire); + } + } + + explicit iterator(harris_michael_list_based_set& list, find_info&& info) : list(&list), info(std::move(info)) {} + + harris_michael_list_based_set* list; + find_info info; +}; + +template +struct harris_michael_list_based_set::node : reclaimer::template enable_concurrent_ptr { + const Key key; + concurrent_ptr next; + template + explicit node(Args&&... args) : key(std::forward(args)...), next() {} +}; + +template +auto harris_michael_list_based_set::iterator::operator++() -> iterator& { + assert(info.cur.get() != nullptr); + auto next = info.cur->next.load(std::memory_order_relaxed); + guard_ptr tmp_guard; + // (1) - this acquire-load synchronizes-with the release-CAS (7, 8, 10, 13) + if (next.mark() == 0 && tmp_guard.acquire_if_equal(info.cur->next, next, std::memory_order_acquire)) { + info.prev = &info.cur->next; + info.save = std::move(info.cur); + info.cur = std::move(tmp_guard); + } else { + // cur is marked for removal + // -> use find to remove it and get to the next node with a compare(key, cur->key) == false + auto key = info.cur->key; + backoff backoff; + list->find(key, info, backoff); + } + assert(info.prev == &list->head || info.cur.get() == nullptr || + (info.save.get() != nullptr && &info.save->next == info.prev)); + return *this; +} + +template +auto harris_michael_list_based_set::iterator::operator++(int) -> iterator { + iterator retval = *this; + ++(*this); + return retval; +} + +template +harris_michael_list_based_set::~harris_michael_list_based_set() { + // delete all remaining nodes + // (3) - this acquire-load synchronizes-with the release-CAS (7, 8, 10, 13) + auto p = head.load(std::memory_order_acquire); + while (p) { + // (4) - this acquire-load synchronizes-with the release-CAS (7, 8, 10, 13) + auto next = p->next.load(std::memory_order_acquire); + delete p.get(); + p = next; + } +} + +template +bool harris_michael_list_based_set::find(const Key& key, find_info& info, backoff& backoff) { + assert((info.save == nullptr && info.prev == &head) || &info.save->next == info.prev); + concurrent_ptr* start = info.prev; + guard_ptr start_guard = info.save; // we have to keep a guard_ptr to prevent start's node from getting reclaimed. +retry: + info.prev = start; + info.save = start_guard; + info.next = info.prev->load(std::memory_order_relaxed); + if (info.next.mark() != 0) { + // our start node is marked for removal -> we have to restart from head + start = &head; + start_guard.reset(); + goto retry; + } + + for (;;) { + // (5) - this acquire-load synchronizes-with the release-CAS (7, 8, 10, 13) + if (!info.cur.acquire_if_equal(*info.prev, info.next, std::memory_order_acquire)) { + goto retry; + } + + if (!info.cur) { + return false; + } + + info.next = info.cur->next.load(std::memory_order_relaxed); + if (info.next.mark() != 0) { + // Node *cur is marked for deletion -> update the link and retire the element + + // (6) - this acquire-load synchronizes-with the release-CAS (7, 8, 10, 13) + info.next = info.cur->next.load(std::memory_order_acquire).get(); + + // Try to splice out node + marked_ptr expected = info.cur.get(); + // (7) - this release-CAS synchronizes with the acquire-load (1, 2, 3, 4, 5, 6, 11) + // and the require-CAS (9, 12) + // it is the head of a potential release sequence containing (9, 12) + if (!info.prev->compare_exchange_weak( + expected, info.next, std::memory_order_release, std::memory_order_relaxed)) { + backoff(); + goto retry; + } + info.cur.reclaim(); + } else { + if (info.prev->load(std::memory_order_relaxed) != info.cur.get()) { + goto retry; // cur might be cut from list. + } + + const Key& ckey = info.cur->key; + compare compare; + if (!compare(ckey, key)) { + return !compare(key, ckey); + } + + info.prev = &info.cur->next; + std::swap(info.save, info.cur); + } + } +} + +template +bool harris_michael_list_based_set::contains(const Key& key) { + find_info info{&head}; + backoff backoff; + return find(key, info, backoff); +} + +template +auto harris_michael_list_based_set::find(const Key& key) -> iterator { + find_info info{&head}; + backoff backoff; + if (find(key, info, backoff)) { + return iterator(*this, std::move(info)); + } + return end(); +} + +template +template +bool harris_michael_list_based_set::emplace(Args&&... args) { + auto result = emplace_or_get(std::forward(args)...); + return result.second; +} + +template +template +auto harris_michael_list_based_set::emplace_or_get(Args&&... args) -> std::pair { + node* n = new node(std::forward(args)...); + find_info info{&head}; + backoff backoff; + for (;;) { + if (find(n->key, info, backoff)) { + delete n; + return {iterator(*this, std::move(info)), false}; + } + + // Try to install new node + marked_ptr expected = info.cur.get(); + n->next.store(expected, std::memory_order_relaxed); + guard_ptr new_guard(n); + + // (8) - this release-CAS synchronizes with the acquire-load (1, 2, 3, 4, 5, 6, 11) + // and the acquire-CAS (9, 12) + // it is the head of a potential release sequence containing (9, 12) + if (info.prev->compare_exchange_weak(expected, n, std::memory_order_release, std::memory_order_relaxed)) { + info.cur = std::move(new_guard); + return {iterator(*this, std::move(info)), true}; + } + + backoff(); + } +} + +template +bool harris_michael_list_based_set::erase(const Key& key) { + backoff backoff; + find_info info{&head}; + // Find node in list with matching key and mark it for reclamation. + for (;;) { + if (!find(key, info, backoff)) { + return false; // No such node in the list + } + + // (9) - this acquire-CAS synchronizes with the release-CAS (7, 8, 10, 13) + // and is part of a release sequence headed by those operations + if (info.cur->next.compare_exchange_weak( + info.next, marked_ptr(info.next.get(), 1), std::memory_order_acquire, std::memory_order_relaxed)) { + break; + } + + backoff(); + } + + assert(info.next.mark() == 0); + assert(info.cur.mark() == 0); + + // Try to splice out node + marked_ptr expected = info.cur; + // (10) - this release-CAS synchronizes with the acquire-load (1, 2, 3, 4, 5, 6, 11) + // and the acquire-CAS (9, 12) + // it is the head of a potential release sequence containing (9, 12) + if (info.prev->compare_exchange_weak(expected, info.next, std::memory_order_release, std::memory_order_relaxed)) { + info.cur.reclaim(); + } else { + // Another thread interfered -> rewalk the list to ensure reclamation of marked node before returning. + find(key, info, backoff); + } + + return true; +} + +template +auto harris_michael_list_based_set::erase(iterator pos) -> iterator { + backoff backoff; + // (11) - this acquire-load synchronizes-with the release-CAS (7, 8, 10, 13) + auto next = pos.info.cur->next.load(std::memory_order_acquire); + while (next.mark() == 0) { + // (12) - this acquire-CAS synchronizes-with the release-CAS (7, 8, 10, 13) + // and is part of a release sequence headed by those operations + if (pos.info.cur->next.compare_exchange_weak(next, marked_ptr(next.get(), 1), std::memory_order_acquire)) { + break; + } + + backoff(); + } + + guard_ptr next_guard(next.get()); + assert(pos.info.cur.mark() == 0); + + // Try to splice out node + marked_ptr expected = pos.info.cur; + // (13) - this release-CAS synchronizes with the acquire-load (1, 2, 3, 4, 5, 6, 11) + // and the acquire-CAS (9, 12) + // it is the head of a potential release sequence containing (9, 12) + if (pos.info.prev->compare_exchange_weak( + expected, next_guard, std::memory_order_release, std::memory_order_relaxed)) { + pos.info.cur.reclaim(); + pos.info.cur = std::move(next_guard); + } else { + next_guard.reset(); + Key key = pos.info.cur->key; + + // Another thread interfered -> rewalk the list to ensure reclamation of marked node before returning. + find(key, pos.info, backoff); + } + + return pos; +} + +template +auto harris_michael_list_based_set::begin() -> iterator { + return iterator(*this, &head); +} + +template +auto harris_michael_list_based_set::end() -> iterator { + return iterator(*this, nullptr); +} +} // namespace xenium + +#endif diff --git a/xenium/hash.hpp b/include/xenium/hash.hpp similarity index 100% rename from xenium/hash.hpp rename to include/xenium/hash.hpp diff --git a/xenium/impl/vyukov_hash_map.hpp b/include/xenium/impl/vyukov_hash_map.hpp similarity index 100% rename from xenium/impl/vyukov_hash_map.hpp rename to include/xenium/impl/vyukov_hash_map.hpp diff --git a/xenium/impl/vyukov_hash_map_traits.hpp b/include/xenium/impl/vyukov_hash_map_traits.hpp similarity index 100% rename from xenium/impl/vyukov_hash_map_traits.hpp rename to include/xenium/impl/vyukov_hash_map_traits.hpp diff --git a/xenium/kirsch_bounded_kfifo_queue.hpp b/include/xenium/kirsch_bounded_kfifo_queue.hpp similarity index 100% rename from xenium/kirsch_bounded_kfifo_queue.hpp rename to include/xenium/kirsch_bounded_kfifo_queue.hpp diff --git a/xenium/kirsch_kfifo_queue.hpp b/include/xenium/kirsch_kfifo_queue.hpp similarity index 100% rename from xenium/kirsch_kfifo_queue.hpp rename to include/xenium/kirsch_kfifo_queue.hpp diff --git a/xenium/left_right.hpp b/include/xenium/left_right.hpp similarity index 100% rename from xenium/left_right.hpp rename to include/xenium/left_right.hpp diff --git a/xenium/marked_ptr.hpp b/include/xenium/marked_ptr.hpp similarity index 96% rename from xenium/marked_ptr.hpp rename to include/xenium/marked_ptr.hpp index 11d355c..33f22d0 100644 --- a/xenium/marked_ptr.hpp +++ b/include/xenium/marked_ptr.hpp @@ -1,179 +1,179 @@ -// -// Copyright (c) 2018-2020 Manuel Pöter. -// Licensed under the MIT License. See LICENSE file in the project root for full license information. -// - -#ifndef XENIUM_MARKED_PTR_HPP -#define XENIUM_MARKED_PTR_HPP - -#include - -#include -#include -#include - -#ifndef XENIUM_MAX_UPPER_MARK_BITS - #define XENIUM_MAX_UPPER_MARK_BITS 16 -#endif - -#ifdef _MSC_VER - #pragma warning(push) - // TODO - remove this after upgrading to C++17 - #pragma warning(disable : 4127) // conditional expression is constant - #pragma warning(disable : 4293) // shift count negative or too big -#endif - -namespace xenium { -/** - * @brief A pointer with an embedded mark/tag value. - * - * Acts like a pointer, but has an embeded mark/tag value with `MarkBits` bits. - * On most systems pointers are only 48-bit, leaving the 16 topmost bits clear. - * Therefore, the mark value is usually embeded in these top-most bits. For mark - * mark values that exceed `MaxUpperMarkBits` bits the remaining bits are embedded - * in the lowest bits, but this requires the pointer to be aligned properly. - * - * @tparam T - * @tparam MarkBits the number of bits used for the mark value - * @tparam MaxUpperMarkBits the max number of bits to be used (defaults to 16) - */ -template -class marked_ptr { - static_assert(MarkBits > 0, "should never happen - compiler should pick the specilization for zero MarkBits!"); - static constexpr uintptr_t pointer_bits = sizeof(T*) * 8 - MarkBits; - static constexpr uintptr_t MarkMask = (static_cast(1) << MarkBits) - 1; - - static constexpr uintptr_t lower_mark_bits = MarkBits < MaxUpperMarkBits ? 0 : MarkBits - MaxUpperMarkBits; - static constexpr uintptr_t upper_mark_bits = MarkBits - lower_mark_bits; - static constexpr uintptr_t pointer_mask = ((static_cast(1) << pointer_bits) - 1) << lower_mark_bits; - -public: - static constexpr uintptr_t number_of_mark_bits = MarkBits; - static_assert(MarkBits <= 32, "MarkBits must not be greater than 32."); - static_assert(sizeof(T*) == 8, "marked_ptr requires 64bit pointers."); - - /** - * @brief Construct a marked_ptr with an optional mark value. - * - * The `mark` value is automatically trimmed to `MarkBits` bits. - */ - marked_ptr(T* p = nullptr, uintptr_t mark = 0) noexcept : _ptr(make_ptr(p, mark)) {} // NOLINT - - /** - * @brief Reset the pointer to `nullptr` and the mark to 0. - */ - void reset() noexcept { _ptr = nullptr; } - - /** - * @brief Get the mark value. - */ - [[nodiscard]] uintptr_t mark() const noexcept { - return utils::rotate::right(reinterpret_cast(_ptr)) >> pointer_bits; - } - - /** - * @brief Get underlying pointer (with mark bits stripped off). - */ - [[nodiscard]] T* get() const noexcept { - auto ip = reinterpret_cast(_ptr); - if constexpr (number_of_mark_bits != 0) { - ip &= pointer_mask; - } - return reinterpret_cast(ip); - } - - /** - * @brief True if `get() != nullptr || mark() != 0` - */ - explicit operator bool() const noexcept { return _ptr != nullptr; } - - /** - * @brief Get pointer with mark bits stripped off. - */ - T* operator->() const noexcept { return get(); } - - /** - * @brief Get reference to target of pointer. - */ - T& operator*() const noexcept { return *get(); } - - inline friend bool operator==(const marked_ptr& l, const marked_ptr& r) { return l._ptr == r._ptr; } - inline friend bool operator!=(const marked_ptr& l, const marked_ptr& r) { return l._ptr != r._ptr; } - -private: - T* make_ptr(T* p, uintptr_t mark) noexcept { - assert((reinterpret_cast(p) & ~pointer_mask) == 0 && - "bits reserved for masking are occupied by the pointer"); - - auto ip = reinterpret_cast(p); - if constexpr (number_of_mark_bits == 0) { - assert(mark == 0); - return p; - } else { - mark = utils::rotate::left(mark << pointer_bits); - return reinterpret_cast(ip | mark); - } - } - - T* _ptr; - -#ifdef _MSC_VER - // These members are only for the VS debugger visualizer (natvis). - enum Masking { MarkMask_ = MarkMask }; - using PtrType = T*; -#endif -}; - -template -class marked_ptr { -public: - static constexpr uintptr_t number_of_mark_bits = 0; - - /** - * @brief Construct a marked_ptr. - */ - marked_ptr(T* p = nullptr) noexcept { _ptr = p; } // NOLINT - - /** - * @brief Reset the pointer to `nullptr` and the mark to 0. - */ - void reset() noexcept { _ptr = nullptr; } - - /** - * @brief Get the mark value. - */ - [[nodiscard]] uintptr_t mark() const noexcept { return 0; } - - /** - * @brief Get underlying pointer. - */ - [[nodiscard]] T* get() const noexcept { return _ptr; } - - /** - * @brief True if `get() != nullptr || mark() != 0` - */ - explicit operator bool() const noexcept { return _ptr != nullptr; } - - /** - * @brief Get pointer with mark bits stripped off. - */ - T* operator->() const noexcept { return get(); } - - /** - * @brief Get reference to target of pointer. - */ - T& operator*() const noexcept { return *get(); } - - inline friend bool operator==(const marked_ptr& l, const marked_ptr& r) { return l._ptr == r._ptr; } - inline friend bool operator!=(const marked_ptr& l, const marked_ptr& r) { return l._ptr != r._ptr; } - -private: - T* _ptr; -}; -} // namespace xenium - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - +// +// Copyright (c) 2018-2020 Manuel Pöter. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +#ifndef XENIUM_MARKED_PTR_HPP +#define XENIUM_MARKED_PTR_HPP + +#include + +#include +#include +#include + +#ifndef XENIUM_MAX_UPPER_MARK_BITS + #define XENIUM_MAX_UPPER_MARK_BITS 16 +#endif + +#ifdef _MSC_VER + #pragma warning(push) + // TODO - remove this after upgrading to C++17 + #pragma warning(disable : 4127) // conditional expression is constant + #pragma warning(disable : 4293) // shift count negative or too big +#endif + +namespace xenium { +/** + * @brief A pointer with an embedded mark/tag value. + * + * Acts like a pointer, but has an embeded mark/tag value with `MarkBits` bits. + * On most systems pointers are only 48-bit, leaving the 16 topmost bits clear. + * Therefore, the mark value is usually embeded in these top-most bits. For mark + * mark values that exceed `MaxUpperMarkBits` bits the remaining bits are embedded + * in the lowest bits, but this requires the pointer to be aligned properly. + * + * @tparam T + * @tparam MarkBits the number of bits used for the mark value + * @tparam MaxUpperMarkBits the max number of bits to be used (defaults to 16) + */ +template +class marked_ptr { + static_assert(MarkBits > 0, "should never happen - compiler should pick the specilization for zero MarkBits!"); + static constexpr uintptr_t pointer_bits = sizeof(T*) * 8 - MarkBits; + static constexpr uintptr_t MarkMask = (static_cast(1) << MarkBits) - 1; + + static constexpr uintptr_t lower_mark_bits = MarkBits < MaxUpperMarkBits ? 0 : MarkBits - MaxUpperMarkBits; + static constexpr uintptr_t upper_mark_bits = MarkBits - lower_mark_bits; + static constexpr uintptr_t pointer_mask = ((static_cast(1) << pointer_bits) - 1) << lower_mark_bits; + +public: + static constexpr uintptr_t number_of_mark_bits = MarkBits; + static_assert(MarkBits <= 32, "MarkBits must not be greater than 32."); + static_assert(sizeof(T*) == 8, "marked_ptr requires 64bit pointers."); + + /** + * @brief Construct a marked_ptr with an optional mark value. + * + * The `mark` value is automatically trimmed to `MarkBits` bits. + */ + marked_ptr(T* p = nullptr, uintptr_t mark = 0) noexcept : _ptr(make_ptr(p, mark)) {} // NOLINT + + /** + * @brief Reset the pointer to `nullptr` and the mark to 0. + */ + void reset() noexcept { _ptr = nullptr; } + + /** + * @brief Get the mark value. + */ + [[nodiscard]] uintptr_t mark() const noexcept { + return utils::rotate::right(reinterpret_cast(_ptr)) >> pointer_bits; + } + + /** + * @brief Get underlying pointer (with mark bits stripped off). + */ + [[nodiscard]] T* get() const noexcept { + auto ip = reinterpret_cast(_ptr); + if constexpr (number_of_mark_bits != 0) { + ip &= pointer_mask; + } + return reinterpret_cast(ip); + } + + /** + * @brief True if `get() != nullptr || mark() != 0` + */ + explicit operator bool() const noexcept { return _ptr != nullptr; } + + /** + * @brief Get pointer with mark bits stripped off. + */ + T* operator->() const noexcept { return get(); } + + /** + * @brief Get reference to target of pointer. + */ + T& operator*() const noexcept { return *get(); } + + inline friend bool operator==(const marked_ptr& l, const marked_ptr& r) { return l._ptr == r._ptr; } + inline friend bool operator!=(const marked_ptr& l, const marked_ptr& r) { return l._ptr != r._ptr; } + +private: + T* make_ptr(T* p, uintptr_t mark) noexcept { + assert((reinterpret_cast(p) & ~pointer_mask) == 0 && + "bits reserved for masking are occupied by the pointer"); + + auto ip = reinterpret_cast(p); + if constexpr (number_of_mark_bits == 0) { + assert(mark == 0); + return p; + } else { + mark = utils::rotate::left(mark << pointer_bits); + return reinterpret_cast(ip | mark); + } + } + + T* _ptr; + +#ifdef _MSC_VER + // These members are only for the VS debugger visualizer (natvis). + enum Masking { MarkMask_ = MarkMask }; + using PtrType = T*; +#endif +}; + +template +class marked_ptr { +public: + static constexpr uintptr_t number_of_mark_bits = 0; + + /** + * @brief Construct a marked_ptr. + */ + marked_ptr(T* p = nullptr) noexcept { _ptr = p; } // NOLINT + + /** + * @brief Reset the pointer to `nullptr` and the mark to 0. + */ + void reset() noexcept { _ptr = nullptr; } + + /** + * @brief Get the mark value. + */ + [[nodiscard]] uintptr_t mark() const noexcept { return 0; } + + /** + * @brief Get underlying pointer. + */ + [[nodiscard]] T* get() const noexcept { return _ptr; } + + /** + * @brief True if `get() != nullptr || mark() != 0` + */ + explicit operator bool() const noexcept { return _ptr != nullptr; } + + /** + * @brief Get pointer with mark bits stripped off. + */ + T* operator->() const noexcept { return get(); } + + /** + * @brief Get reference to target of pointer. + */ + T& operator*() const noexcept { return *get(); } + + inline friend bool operator==(const marked_ptr& l, const marked_ptr& r) { return l._ptr == r._ptr; } + inline friend bool operator!=(const marked_ptr& l, const marked_ptr& r) { return l._ptr != r._ptr; } + +private: + T* _ptr; +}; +} // namespace xenium + +#ifdef _MSC_VER + #pragma warning(pop) +#endif + #endif \ No newline at end of file diff --git a/xenium/meta.hpp b/include/xenium/meta.hpp similarity index 100% rename from xenium/meta.hpp rename to include/xenium/meta.hpp diff --git a/xenium/michael_scott_queue.hpp b/include/xenium/michael_scott_queue.hpp similarity index 100% rename from xenium/michael_scott_queue.hpp rename to include/xenium/michael_scott_queue.hpp diff --git a/xenium/nikolaev_bounded_queue.hpp b/include/xenium/nikolaev_bounded_queue.hpp similarity index 100% rename from xenium/nikolaev_bounded_queue.hpp rename to include/xenium/nikolaev_bounded_queue.hpp diff --git a/xenium/nikolaev_queue.hpp b/include/xenium/nikolaev_queue.hpp similarity index 100% rename from xenium/nikolaev_queue.hpp rename to include/xenium/nikolaev_queue.hpp diff --git a/xenium/parameter.hpp b/include/xenium/parameter.hpp similarity index 100% rename from xenium/parameter.hpp rename to include/xenium/parameter.hpp diff --git a/xenium/policy.hpp b/include/xenium/policy.hpp similarity index 100% rename from xenium/policy.hpp rename to include/xenium/policy.hpp diff --git a/xenium/ramalhete_queue.hpp b/include/xenium/ramalhete_queue.hpp similarity index 100% rename from xenium/ramalhete_queue.hpp rename to include/xenium/ramalhete_queue.hpp diff --git a/xenium/reclamation/detail/allocation_tracker.hpp b/include/xenium/reclamation/detail/allocation_tracker.hpp similarity index 100% rename from xenium/reclamation/detail/allocation_tracker.hpp rename to include/xenium/reclamation/detail/allocation_tracker.hpp diff --git a/xenium/reclamation/detail/concurrent_ptr.hpp b/include/xenium/reclamation/detail/concurrent_ptr.hpp similarity index 97% rename from xenium/reclamation/detail/concurrent_ptr.hpp rename to include/xenium/reclamation/detail/concurrent_ptr.hpp index 311e02e..c2392bd 100644 --- a/xenium/reclamation/detail/concurrent_ptr.hpp +++ b/include/xenium/reclamation/detail/concurrent_ptr.hpp @@ -1,96 +1,96 @@ -// -// Copyright (c) 2018-2020 Manuel Pöter. -// Licensed under the MIT License. See LICENSE file in the project root for full license information. -// - -#ifndef XENIUM_DETAL_CONCURRENT_PTR_HPP -#define XENIUM_DETAL_CONCURRENT_PTR_HPP - -#include - -#include - -namespace xenium::reclamation::detail { - -//! T must be derived from enable_concurrent_ptr. D is a deleter. -template class GuardPtr> -class concurrent_ptr { -public: - using marked_ptr = xenium::marked_ptr; - using guard_ptr = GuardPtr; - - concurrent_ptr(const marked_ptr& p = marked_ptr()) noexcept : _ptr(p) {} // NOLINT - concurrent_ptr(const concurrent_ptr&) = delete; - concurrent_ptr(concurrent_ptr&&) = delete; - concurrent_ptr& operator=(const concurrent_ptr&) = delete; - concurrent_ptr& operator=(concurrent_ptr&&) = delete; - - // Atomic load that does not guard target from being reclaimed. - [[nodiscard]] marked_ptr load(std::memory_order order = std::memory_order_seq_cst) const { return _ptr.load(order); } - - // Atomic store. - void store(const marked_ptr& src, std::memory_order order = std::memory_order_seq_cst) { _ptr.store(src, order); } - - // Shorthand for store (src.get()) - void store(const guard_ptr& src, std::memory_order order = std::memory_order_seq_cst) { - _ptr.store(src.get(), order); - } - - bool compare_exchange_weak(marked_ptr& expected, - marked_ptr desired, - std::memory_order order = std::memory_order_seq_cst) { - return _ptr.compare_exchange_weak(expected, desired, order); - } - - bool compare_exchange_weak(marked_ptr& expected, - marked_ptr desired, - std::memory_order order = std::memory_order_seq_cst) volatile { - return _ptr.compare_exchange_weak(expected, desired, order); - } - - bool compare_exchange_weak(marked_ptr& expected, - marked_ptr desired, - std::memory_order success, - std::memory_order failure) { - return _ptr.compare_exchange_weak(expected, desired, success, failure); - } - - bool compare_exchange_weak(marked_ptr& expected, - marked_ptr desired, - std::memory_order success, - std::memory_order failure) volatile { - return _ptr.compare_exchange_weak(expected, desired, success, failure); - } - - bool compare_exchange_strong(marked_ptr& expected, - marked_ptr desired, - std::memory_order order = std::memory_order_seq_cst) { - return _ptr.compare_exchange_strong(expected, desired, order); - } - - bool compare_exchange_strong(marked_ptr& expected, - marked_ptr desired, - std::memory_order order = std::memory_order_seq_cst) volatile { - return _ptr.compare_exchange_strong(expected, desired, order); - } - - bool compare_exchange_strong(marked_ptr& expected, - marked_ptr desired, - std::memory_order success, - std::memory_order failure) { - return _ptr.compare_exchange_strong(expected, desired, success, failure); - } - - bool compare_exchange_strong(marked_ptr& expected, - marked_ptr desired, - std::memory_order success, - std::memory_order failure) volatile { - return _ptr.compare_exchange_strong(expected, desired, success, failure); - } - -private: - std::atomic _ptr; -}; -} // namespace xenium::reclamation::detail - -#endif +// +// Copyright (c) 2018-2020 Manuel Pöter. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +#ifndef XENIUM_DETAL_CONCURRENT_PTR_HPP +#define XENIUM_DETAL_CONCURRENT_PTR_HPP + +#include + +#include + +namespace xenium::reclamation::detail { + +//! T must be derived from enable_concurrent_ptr. D is a deleter. +template class GuardPtr> +class concurrent_ptr { +public: + using marked_ptr = xenium::marked_ptr; + using guard_ptr = GuardPtr; + + concurrent_ptr(const marked_ptr& p = marked_ptr()) noexcept : _ptr(p) {} // NOLINT + concurrent_ptr(const concurrent_ptr&) = delete; + concurrent_ptr(concurrent_ptr&&) = delete; + concurrent_ptr& operator=(const concurrent_ptr&) = delete; + concurrent_ptr& operator=(concurrent_ptr&&) = delete; + + // Atomic load that does not guard target from being reclaimed. + [[nodiscard]] marked_ptr load(std::memory_order order = std::memory_order_seq_cst) const { return _ptr.load(order); } + + // Atomic store. + void store(const marked_ptr& src, std::memory_order order = std::memory_order_seq_cst) { _ptr.store(src, order); } + + // Shorthand for store (src.get()) + void store(const guard_ptr& src, std::memory_order order = std::memory_order_seq_cst) { + _ptr.store(src.get(), order); + } + + bool compare_exchange_weak(marked_ptr& expected, + marked_ptr desired, + std::memory_order order = std::memory_order_seq_cst) { + return _ptr.compare_exchange_weak(expected, desired, order); + } + + bool compare_exchange_weak(marked_ptr& expected, + marked_ptr desired, + std::memory_order order = std::memory_order_seq_cst) volatile { + return _ptr.compare_exchange_weak(expected, desired, order); + } + + bool compare_exchange_weak(marked_ptr& expected, + marked_ptr desired, + std::memory_order success, + std::memory_order failure) { + return _ptr.compare_exchange_weak(expected, desired, success, failure); + } + + bool compare_exchange_weak(marked_ptr& expected, + marked_ptr desired, + std::memory_order success, + std::memory_order failure) volatile { + return _ptr.compare_exchange_weak(expected, desired, success, failure); + } + + bool compare_exchange_strong(marked_ptr& expected, + marked_ptr desired, + std::memory_order order = std::memory_order_seq_cst) { + return _ptr.compare_exchange_strong(expected, desired, order); + } + + bool compare_exchange_strong(marked_ptr& expected, + marked_ptr desired, + std::memory_order order = std::memory_order_seq_cst) volatile { + return _ptr.compare_exchange_strong(expected, desired, order); + } + + bool compare_exchange_strong(marked_ptr& expected, + marked_ptr desired, + std::memory_order success, + std::memory_order failure) { + return _ptr.compare_exchange_strong(expected, desired, success, failure); + } + + bool compare_exchange_strong(marked_ptr& expected, + marked_ptr desired, + std::memory_order success, + std::memory_order failure) volatile { + return _ptr.compare_exchange_strong(expected, desired, success, failure); + } + +private: + std::atomic _ptr; +}; +} // namespace xenium::reclamation::detail + +#endif diff --git a/xenium/reclamation/detail/deletable_object.hpp b/include/xenium/reclamation/detail/deletable_object.hpp similarity index 96% rename from xenium/reclamation/detail/deletable_object.hpp rename to include/xenium/reclamation/detail/deletable_object.hpp index 5ba2fe8..a7847a3 100644 --- a/xenium/reclamation/detail/deletable_object.hpp +++ b/include/xenium/reclamation/detail/deletable_object.hpp @@ -1,76 +1,76 @@ -// -// Copyright (c) 2018-2020 Manuel Pöter. -// Licensed under the MIT License. See LICENSE file in the project root for full license information. -// - -#ifndef XENIUM_DETAIL_DELETABLE_OBJECT_HPP -#define XENIUM_DETAIL_DELETABLE_OBJECT_HPP - -#include -#include - -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 26495) // uninitialized member variable -#endif - -namespace xenium::reclamation::detail { - -struct deletable_object { - virtual void delete_self() = 0; - deletable_object* next = nullptr; - -protected: - virtual ~deletable_object() = default; -}; - -inline void delete_objects(deletable_object*& list) { - auto* cur = list; - for (deletable_object* next = nullptr; cur != nullptr; cur = next) { - next = cur->next; - cur->delete_self(); - } - list = nullptr; -} - -template -struct deletable_object_with_non_empty_deleter : Base { - using Deleter = DeleterT; - void delete_self() override { - auto& my_deleter = reinterpret_cast(_deleter_buffer); - Deleter deleter(std::move(my_deleter)); - my_deleter.~Deleter(); // NOLINT (use-after-move) - - deleter(static_cast(this)); - } - - void set_deleter(Deleter deleter) { new (&_deleter_buffer) Deleter(std::move(deleter)); } - -private: - using buffer = typename std::aligned_storage::type; - buffer _deleter_buffer; -}; - -template -struct deletable_object_with_empty_deleter : Base { - using Deleter = DeleterT; - void delete_self() override { - static_assert(std::is_default_constructible::value, "empty deleters must be default constructible"); - Deleter deleter{}; - deleter(static_cast(this)); - } - - void set_deleter(Deleter /*deleter*/) {} -}; - -template , class Base = deletable_object> -using deletable_object_impl = std::conditional_t::value, - deletable_object_with_empty_deleter, - deletable_object_with_non_empty_deleter>; -} // namespace xenium::reclamation::detail - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - -#endif +// +// Copyright (c) 2018-2020 Manuel Pöter. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +#ifndef XENIUM_DETAIL_DELETABLE_OBJECT_HPP +#define XENIUM_DETAIL_DELETABLE_OBJECT_HPP + +#include +#include + +#ifdef _MSC_VER + #pragma warning(push) + #pragma warning(disable : 26495) // uninitialized member variable +#endif + +namespace xenium::reclamation::detail { + +struct deletable_object { + virtual void delete_self() = 0; + deletable_object* next = nullptr; + +protected: + virtual ~deletable_object() = default; +}; + +inline void delete_objects(deletable_object*& list) { + auto* cur = list; + for (deletable_object* next = nullptr; cur != nullptr; cur = next) { + next = cur->next; + cur->delete_self(); + } + list = nullptr; +} + +template +struct deletable_object_with_non_empty_deleter : Base { + using Deleter = DeleterT; + void delete_self() override { + auto& my_deleter = reinterpret_cast(_deleter_buffer); + Deleter deleter(std::move(my_deleter)); + my_deleter.~Deleter(); // NOLINT (use-after-move) + + deleter(static_cast(this)); + } + + void set_deleter(Deleter deleter) { new (&_deleter_buffer) Deleter(std::move(deleter)); } + +private: + using buffer = typename std::aligned_storage::type; + buffer _deleter_buffer; +}; + +template +struct deletable_object_with_empty_deleter : Base { + using Deleter = DeleterT; + void delete_self() override { + static_assert(std::is_default_constructible::value, "empty deleters must be default constructible"); + Deleter deleter{}; + deleter(static_cast(this)); + } + + void set_deleter(Deleter /*deleter*/) {} +}; + +template , class Base = deletable_object> +using deletable_object_impl = std::conditional_t::value, + deletable_object_with_empty_deleter, + deletable_object_with_non_empty_deleter>; +} // namespace xenium::reclamation::detail + +#ifdef _MSC_VER + #pragma warning(pop) +#endif + +#endif diff --git a/xenium/reclamation/detail/guard_ptr.hpp b/include/xenium/reclamation/detail/guard_ptr.hpp similarity index 96% rename from xenium/reclamation/detail/guard_ptr.hpp rename to include/xenium/reclamation/detail/guard_ptr.hpp index 3c7f6ee..3423337 100644 --- a/xenium/reclamation/detail/guard_ptr.hpp +++ b/include/xenium/reclamation/detail/guard_ptr.hpp @@ -1,53 +1,53 @@ -// -// Copyright (c) 2018-2020 Manuel Pöter. -// Licensed under the MIT License. See LICENSE file in the project root for full license information. -// - -#ifndef XENIUM_DETAL_GUARD_PTR_HPP -#define XENIUM_DETAL_GUARD_PTR_HPP - -#include - -namespace xenium::reclamation::detail { - -template -class guard_ptr { -public: - ~guard_ptr() { self().reset(); } - - // Get underlying pointer - [[nodiscard]] T* get() const noexcept { return ptr.get(); } - - // Get mark bits - [[nodiscard]] uintptr_t mark() const noexcept { return ptr.mark(); } - - operator MarkedPtr() const noexcept { return ptr; } // NOLINT (explicit) - - // True if get() != nullptr || mark() != 0 - explicit operator bool() const noexcept { return static_cast(ptr); } - - // Get pointer with mark bits stripped off. Undefined if target has been reclaimed. - T* operator->() const noexcept { return ptr.get(); } - - // Get reference to target of pointer. Undefined if target has been reclaimed. - T& operator*() const noexcept { return *ptr; } - - // Swap two guards - void swap(Derived& g) noexcept { - std::swap(ptr, g.ptr); - self().do_swap(g); - } - -protected: - // NOLINTNEXTLINE (explicit-constructor) - guard_ptr(const MarkedPtr& p = MarkedPtr{}) noexcept : ptr(p) {} - MarkedPtr ptr; - - void do_swap(Derived&) noexcept {} // empty dummy - -private: - Derived& self() { return static_cast(*this); } -}; -} // namespace xenium::reclamation::detail - -#endif +// +// Copyright (c) 2018-2020 Manuel Pöter. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +#ifndef XENIUM_DETAL_GUARD_PTR_HPP +#define XENIUM_DETAL_GUARD_PTR_HPP + +#include + +namespace xenium::reclamation::detail { + +template +class guard_ptr { +public: + ~guard_ptr() { self().reset(); } + + // Get underlying pointer + [[nodiscard]] T* get() const noexcept { return ptr.get(); } + + // Get mark bits + [[nodiscard]] uintptr_t mark() const noexcept { return ptr.mark(); } + + operator MarkedPtr() const noexcept { return ptr; } // NOLINT (explicit) + + // True if get() != nullptr || mark() != 0 + explicit operator bool() const noexcept { return static_cast(ptr); } + + // Get pointer with mark bits stripped off. Undefined if target has been reclaimed. + T* operator->() const noexcept { return ptr.get(); } + + // Get reference to target of pointer. Undefined if target has been reclaimed. + T& operator*() const noexcept { return *ptr; } + + // Swap two guards + void swap(Derived& g) noexcept { + std::swap(ptr, g.ptr); + self().do_swap(g); + } + +protected: + // NOLINTNEXTLINE (explicit-constructor) + guard_ptr(const MarkedPtr& p = MarkedPtr{}) noexcept : ptr(p) {} + MarkedPtr ptr; + + void do_swap(Derived&) noexcept {} // empty dummy + +private: + Derived& self() { return static_cast(*this); } +}; +} // namespace xenium::reclamation::detail + +#endif diff --git a/xenium/reclamation/detail/orphan.hpp b/include/xenium/reclamation/detail/orphan.hpp similarity index 96% rename from xenium/reclamation/detail/orphan.hpp rename to include/xenium/reclamation/detail/orphan.hpp index ece6c04..ebe6501 100644 --- a/xenium/reclamation/detail/orphan.hpp +++ b/include/xenium/reclamation/detail/orphan.hpp @@ -1,34 +1,34 @@ -// -// Copyright (c) 2018-2020 Manuel Pöter. -// Licensed under the MIT License. See LICENSE file in the project root for full license information. -// - -#ifndef XENIUM_DETAIL_ORPHAN_HPP -#define XENIUM_DETAIL_ORPHAN_HPP - -#include -#include - -namespace xenium::reclamation::detail { - -template -struct orphan : detail::deletable_object_impl> { - orphan(unsigned target_epoch, std::array& retire_lists) : - target_epoch(target_epoch), - retire_lists(retire_lists) {} - - ~orphan() override { - for (auto p : retire_lists) { - detail::delete_objects(p); - } - } - - const unsigned target_epoch; - -private: - std::array retire_lists; -}; - -} // namespace xenium::reclamation::detail - -#endif +// +// Copyright (c) 2018-2020 Manuel Pöter. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +#ifndef XENIUM_DETAIL_ORPHAN_HPP +#define XENIUM_DETAIL_ORPHAN_HPP + +#include +#include + +namespace xenium::reclamation::detail { + +template +struct orphan : detail::deletable_object_impl> { + orphan(unsigned target_epoch, std::array& retire_lists) : + target_epoch(target_epoch), + retire_lists(retire_lists) {} + + ~orphan() override { + for (auto p : retire_lists) { + detail::delete_objects(p); + } + } + + const unsigned target_epoch; + +private: + std::array retire_lists; +}; + +} // namespace xenium::reclamation::detail + +#endif diff --git a/xenium/reclamation/detail/perf_counter.hpp b/include/xenium/reclamation/detail/perf_counter.hpp similarity index 100% rename from xenium/reclamation/detail/perf_counter.hpp rename to include/xenium/reclamation/detail/perf_counter.hpp diff --git a/xenium/reclamation/detail/retire_list.hpp b/include/xenium/reclamation/detail/retire_list.hpp similarity index 100% rename from xenium/reclamation/detail/retire_list.hpp rename to include/xenium/reclamation/detail/retire_list.hpp diff --git a/xenium/reclamation/detail/thread_block_list.hpp b/include/xenium/reclamation/detail/thread_block_list.hpp similarity index 96% rename from xenium/reclamation/detail/thread_block_list.hpp rename to include/xenium/reclamation/detail/thread_block_list.hpp index cd3890c..a95d818 100644 --- a/xenium/reclamation/detail/thread_block_list.hpp +++ b/include/xenium/reclamation/detail/thread_block_list.hpp @@ -1,190 +1,190 @@ -// -// Copyright (c) 2018-2020 Manuel Pöter. -// Licensed under the MIT License. See LICENSE file in the project root for full license information. -// - -#ifndef XENIUM_DETAIL_THREAD_BLOCK_LIST_HPP -#define XENIUM_DETAIL_THREAD_BLOCK_LIST_HPP - -#include -#include - -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4324) // structure was padded due to alignment specifier -#endif - -namespace xenium::reclamation::detail { - -template -class thread_block_list { - enum class entry_state { free, inactive, active }; - -public: - struct entry { - entry() : next_entry(nullptr), state(entry_state::active) {} - - // Normally this load operation can use relaxed semantic, as all reclamation schemes - // that use it have an acquire-fence that is sequenced-after calling is_active. - // However, TSan does not support acquire-fences, so in order to avoid false - // positives we have to allow other memory orders as well. - [[nodiscard]] bool is_active(std::memory_order memory_order = std::memory_order_relaxed) const { - return state.load(memory_order) == entry_state::active; - } - - void abandon() { - assert(is_active()); - // (1) - this release-store synchronizes-with the acquire-CAS (2) - // or any acquire-fence that is sequenced-after calling is_active. - state.store(entry_state::free, std::memory_order_release); - } - - void activate() { - assert(state.load(std::memory_order_relaxed) == entry_state::inactive); - state.store(entry_state::active, std::memory_order_release); - } - - private: - friend class thread_block_list; - - bool try_adopt(entry_state initial_state) { - if (state.load(std::memory_order_relaxed) == entry_state::free) { - auto expected = entry_state::free; - // (2) - this acquire-CAS synchronizes-with the release-store (1) - return state.compare_exchange_strong(expected, initial_state, std::memory_order_acquire); - } - return false; - } - - // next_entry is only set once when it gets inserted into the list and is never changed afterwards - // -> therefore it does not have to be atomic - T* next_entry; - - // state is used to manage ownership and active status of entries - std::atomic state; - }; - - class iterator { - T* ptr = nullptr; - - explicit iterator(T* ptr) : ptr(ptr) {} - - public: - using iterator_category = std::forward_iterator_tag; - using value_type = T; - using difference_type = std::ptrdiff_t; - using pointer = T*; - using reference = T&; - - iterator() = default; - - void swap(iterator& other) noexcept { std::swap(ptr, other.ptr); } - - iterator& operator++() { - assert(ptr != nullptr); - ptr = ptr->next_entry; - return *this; - } - - iterator operator++(int) { - assert(ptr != nullptr); - iterator tmp(*this); - ptr = ptr->next_entry; - return tmp; - } - - bool operator==(const iterator& rhs) const { return ptr == rhs.ptr; } - - bool operator!=(const iterator& rhs) const { return ptr != rhs.ptr; } - - T& operator*() const { - assert(ptr != nullptr); - return *ptr; - } - - T* operator->() const { - assert(ptr != nullptr); - return ptr; - } - - friend class thread_block_list; - }; - - T* acquire_entry() { return adopt_or_create_entry(entry_state::active); } - - T* acquire_inactive_entry() { return adopt_or_create_entry(entry_state::inactive); } - - void release_entry(T* entry) { entry->abandon(); } - - iterator begin() { - // (3) - this acquire-load synchronizes-with the release-CAS (6) - return iterator{head.load(std::memory_order_acquire)}; - } - - iterator end() { return iterator{}; } - - void abandon_retired_nodes(DeletableObject* obj) { - auto* last = obj; - auto* next = last->next; - while (next) { - last = next; - next = last->next; - } - - auto* h = abandoned_retired_nodes.load(std::memory_order_relaxed); - do { - last->next = h; - // (4) - this releas-CAS synchronizes-with the acquire-exchange (5) - } while ( - !abandoned_retired_nodes.compare_exchange_weak(h, obj, std::memory_order_release, std::memory_order_relaxed)); - } - - DeletableObject* adopt_abandoned_retired_nodes() { - if (abandoned_retired_nodes.load(std::memory_order_relaxed) == nullptr) { - return nullptr; - } - - // (5) - this acquire-exchange synchronizes-with the release-CAS (4) - return abandoned_retired_nodes.exchange(nullptr, std::memory_order_acquire); - } - -private: - void add_entry(T* node) { - auto* h = head.load(std::memory_order_relaxed); - do { - node->next_entry = h; - // (6) - this release-CAS synchronizes-with the acquire-loads (3, 7) - } while (!head.compare_exchange_weak(h, node, std::memory_order_release, std::memory_order_relaxed)); - } - - T* adopt_or_create_entry(entry_state initial_state) { - static_assert(std::is_base_of::value, "T must derive from entry."); - - // (7) - this acquire-load synchronizes-with the release-CAS (6) - T* result = head.load(std::memory_order_acquire); - while (result) { - if (result->try_adopt(initial_state)) { - return result; - } - - result = result->next_entry; - } - - result = new T(); - result->state.store(initial_state, std::memory_order_relaxed); - add_entry(result); - return result; - } - - std::atomic head{nullptr}; - - alignas(64) std::atomic abandoned_retired_nodes{nullptr}; -}; - -} // namespace xenium::reclamation::detail - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - -#endif +// +// Copyright (c) 2018-2020 Manuel Pöter. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +#ifndef XENIUM_DETAIL_THREAD_BLOCK_LIST_HPP +#define XENIUM_DETAIL_THREAD_BLOCK_LIST_HPP + +#include +#include + +#ifdef _MSC_VER + #pragma warning(push) + #pragma warning(disable : 4324) // structure was padded due to alignment specifier +#endif + +namespace xenium::reclamation::detail { + +template +class thread_block_list { + enum class entry_state { free, inactive, active }; + +public: + struct entry { + entry() : next_entry(nullptr), state(entry_state::active) {} + + // Normally this load operation can use relaxed semantic, as all reclamation schemes + // that use it have an acquire-fence that is sequenced-after calling is_active. + // However, TSan does not support acquire-fences, so in order to avoid false + // positives we have to allow other memory orders as well. + [[nodiscard]] bool is_active(std::memory_order memory_order = std::memory_order_relaxed) const { + return state.load(memory_order) == entry_state::active; + } + + void abandon() { + assert(is_active()); + // (1) - this release-store synchronizes-with the acquire-CAS (2) + // or any acquire-fence that is sequenced-after calling is_active. + state.store(entry_state::free, std::memory_order_release); + } + + void activate() { + assert(state.load(std::memory_order_relaxed) == entry_state::inactive); + state.store(entry_state::active, std::memory_order_release); + } + + private: + friend class thread_block_list; + + bool try_adopt(entry_state initial_state) { + if (state.load(std::memory_order_relaxed) == entry_state::free) { + auto expected = entry_state::free; + // (2) - this acquire-CAS synchronizes-with the release-store (1) + return state.compare_exchange_strong(expected, initial_state, std::memory_order_acquire); + } + return false; + } + + // next_entry is only set once when it gets inserted into the list and is never changed afterwards + // -> therefore it does not have to be atomic + T* next_entry; + + // state is used to manage ownership and active status of entries + std::atomic state; + }; + + class iterator { + T* ptr = nullptr; + + explicit iterator(T* ptr) : ptr(ptr) {} + + public: + using iterator_category = std::forward_iterator_tag; + using value_type = T; + using difference_type = std::ptrdiff_t; + using pointer = T*; + using reference = T&; + + iterator() = default; + + void swap(iterator& other) noexcept { std::swap(ptr, other.ptr); } + + iterator& operator++() { + assert(ptr != nullptr); + ptr = ptr->next_entry; + return *this; + } + + iterator operator++(int) { + assert(ptr != nullptr); + iterator tmp(*this); + ptr = ptr->next_entry; + return tmp; + } + + bool operator==(const iterator& rhs) const { return ptr == rhs.ptr; } + + bool operator!=(const iterator& rhs) const { return ptr != rhs.ptr; } + + T& operator*() const { + assert(ptr != nullptr); + return *ptr; + } + + T* operator->() const { + assert(ptr != nullptr); + return ptr; + } + + friend class thread_block_list; + }; + + T* acquire_entry() { return adopt_or_create_entry(entry_state::active); } + + T* acquire_inactive_entry() { return adopt_or_create_entry(entry_state::inactive); } + + void release_entry(T* entry) { entry->abandon(); } + + iterator begin() { + // (3) - this acquire-load synchronizes-with the release-CAS (6) + return iterator{head.load(std::memory_order_acquire)}; + } + + iterator end() { return iterator{}; } + + void abandon_retired_nodes(DeletableObject* obj) { + auto* last = obj; + auto* next = last->next; + while (next) { + last = next; + next = last->next; + } + + auto* h = abandoned_retired_nodes.load(std::memory_order_relaxed); + do { + last->next = h; + // (4) - this releas-CAS synchronizes-with the acquire-exchange (5) + } while ( + !abandoned_retired_nodes.compare_exchange_weak(h, obj, std::memory_order_release, std::memory_order_relaxed)); + } + + DeletableObject* adopt_abandoned_retired_nodes() { + if (abandoned_retired_nodes.load(std::memory_order_relaxed) == nullptr) { + return nullptr; + } + + // (5) - this acquire-exchange synchronizes-with the release-CAS (4) + return abandoned_retired_nodes.exchange(nullptr, std::memory_order_acquire); + } + +private: + void add_entry(T* node) { + auto* h = head.load(std::memory_order_relaxed); + do { + node->next_entry = h; + // (6) - this release-CAS synchronizes-with the acquire-loads (3, 7) + } while (!head.compare_exchange_weak(h, node, std::memory_order_release, std::memory_order_relaxed)); + } + + T* adopt_or_create_entry(entry_state initial_state) { + static_assert(std::is_base_of::value, "T must derive from entry."); + + // (7) - this acquire-load synchronizes-with the release-CAS (6) + T* result = head.load(std::memory_order_acquire); + while (result) { + if (result->try_adopt(initial_state)) { + return result; + } + + result = result->next_entry; + } + + result = new T(); + result->state.store(initial_state, std::memory_order_relaxed); + add_entry(result); + return result; + } + + std::atomic head{nullptr}; + + alignas(64) std::atomic abandoned_retired_nodes{nullptr}; +}; + +} // namespace xenium::reclamation::detail + +#ifdef _MSC_VER + #pragma warning(pop) +#endif + +#endif diff --git a/xenium/reclamation/generic_epoch_based.hpp b/include/xenium/reclamation/generic_epoch_based.hpp similarity index 100% rename from xenium/reclamation/generic_epoch_based.hpp rename to include/xenium/reclamation/generic_epoch_based.hpp diff --git a/xenium/reclamation/hazard_eras.hpp b/include/xenium/reclamation/hazard_eras.hpp similarity index 97% rename from xenium/reclamation/hazard_eras.hpp rename to include/xenium/reclamation/hazard_eras.hpp index 5562c48..791bb08 100644 --- a/xenium/reclamation/hazard_eras.hpp +++ b/include/xenium/reclamation/hazard_eras.hpp @@ -1,276 +1,276 @@ -// -// Copyright (c) 2018-2020 Manuel Pöter. -// Licensed under the MIT License. See LICENSE file in the project root for full license information. -// - -#ifndef XENIUM_HAZARD_ERAS_HPP -#define XENIUM_HAZARD_ERAS_HPP - -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include - -namespace xenium::reclamation { - -/** - * @brief This exception is thrown if a thread tries to allocate a new hazard era, but - * the number of available hazard eras is exhausted. This can only happen when - * `xenium::reclamation::he_allocation::static_strategy` is used. - */ -class bad_hazard_era_alloc : public std::runtime_error { - using std::runtime_error::runtime_error; -}; - -namespace detail { - struct deletable_object_with_eras; - - template - struct basic_he_thread_control_block; - - template class ThreadControlBlock> - struct generic_hazard_era_allocation_strategy { - static constexpr size_t K = K_; - - static size_t retired_nodes_threshold() { return A * number_of_active_hazard_eras() + B; } - - static size_t number_of_active_hazard_eras() { return number_of_active_hes.load(std::memory_order_relaxed); } - - using thread_control_block = ThreadControlBlock; - - private: - friend thread_control_block; - friend basic_he_thread_control_block; - - inline static std::atomic number_of_active_hes{0}; - }; - - template - struct static_he_thread_control_block; - - template - struct dynamic_he_thread_control_block; -} // namespace detail - -namespace he_allocation { - /** - * @brief Hazard era allocation strategy for a static number of hazard eras per thread. - * - * The threshold for the number of retired nodes is calculated as `A * K * num_threads + B`; - * `K`, `A` and `B` can be configured via template parameter. - * - * Note that in contrast to the `hazard_pointer` reclamation scheme, a `hazard_eras::guard_ptr` - * may use the same hazard era entry as a previously created `guard_ptr` if they observe the - * same era. However, since this is not guarenteed, one has to use the same upper bound for the - * number of hazard eras as when using the `hazard_pointer` scheme. - * - * @tparam K The max. number of hazard eras that can be allocated at the same time. - * @tparam A - * @tparam B - */ - template - struct static_strategy : - detail::generic_hazard_era_allocation_strategy {}; - - /** - * @brief Hazard era allocation strategy for a dynamic number of hazard eras per thread. - * - * This strategy uses a linked list of segments for the hazard eras. The first segment can - * hold `K` hazard eras; subsequently allocated segments can hold 1.5x the number of hazard - * eras as the sum of all previous segments. - * - * The threshold for the number of retired nodes is calculated as `A * available_hes + B`, where - * `available_hes` is the max. number of hazard eras that could be allocated by all threads - * at that time, without growing the number of segments. E.g., if `K = 2` and we have two threads - * each with a single segment, then `available_hes = 4`; if one thread later allocates additional - * hes and increases the number of segments such that they can hold up to 10 hazard eras, - * then `available_hes = 10+2 = 12`. - * - * `K`, `A` and `B` can be configured via template parameter. - * - * @tparam K The initial number of hazard eras (i.e., the number of hes that can be allocated - * without the need to grow). - * @tparam A - * @tparam B - */ - template - struct dynamic_strategy : - detail::generic_hazard_era_allocation_strategy {}; -} // namespace he_allocation - -template > -struct hazard_era_traits { - using allocation_strategy = AllocationStrategy; - - template - using with = hazard_era_traits>; -}; - -/** - * @brief An implementation of the hazard eras scheme proposed by Ramalhete and Correia - * \[[RC17](index.html#ref-ramalhete-2017)\]. - * - * For general information about the interface of the reclamation scheme see @ref reclamation_schemes. - * - * This class does not take a list of policies, but a `Traits` type that can be customized - * with a list of policies. The following policies are supported: - * * `xenium::policy::allocation_strategy`
- * Defines how hazard eras are allocated and how the threshold for the number of - * retired nodes is calculated. - * Possible arguments are `xenium::reclamation::he_allocation::static_strategy` - * and 'xenium::reclamation::he_allocation::dynamic_strategy`, where both strategies - * can be further customized via their respective template parameters. - * (defaults to `xenium::reclamation::he_allocation::static_strategy<3>`) - * - * @tparam Traits - */ -template > -class hazard_eras { - using allocation_strategy = typename Traits::allocation_strategy; - using thread_control_block = typename allocation_strategy::thread_control_block; - friend detail::basic_he_thread_control_block; - - template - class guard_ptr; - -public: - /** - * @brief Customize the reclamation scheme with the given policies. - * - * The given policies are applied to the current configuration, replacing previously - * specified policies of the same type. - * - * The resulting type is the newly configured reclamation scheme. - * - * @tparam Policies list of policies to customize the behaviour - */ - template - using with = hazard_eras>; - - template > - class enable_concurrent_ptr; - - class region_guard {}; - - template - using concurrent_ptr = detail::concurrent_ptr; - - ALLOCATION_TRACKER; - -private: - struct thread_data; - - friend struct detail::deletable_object_with_eras; - - using era_t = uint64_t; - inline static std::atomic era_clock{1}; - inline static detail::thread_block_list - global_thread_block_list{}; - static thread_data& local_thread_data(); - - ALLOCATION_TRACKING_FUNCTIONS; -}; - -template -template -class hazard_eras::enable_concurrent_ptr : - public detail::deletable_object_impl, - private detail::tracked_object { -public: - static constexpr std::size_t number_of_mark_bits = N; - -protected: - enable_concurrent_ptr() noexcept { this->construction_era = era_clock.load(std::memory_order_relaxed); } - enable_concurrent_ptr(const enable_concurrent_ptr&) noexcept = default; - enable_concurrent_ptr(enable_concurrent_ptr&&) noexcept = default; - enable_concurrent_ptr& operator=(const enable_concurrent_ptr&) noexcept = default; - enable_concurrent_ptr& operator=(enable_concurrent_ptr&&) noexcept = default; - ~enable_concurrent_ptr() noexcept override = default; - -private: - friend detail::deletable_object_impl; - - template - friend class guard_ptr; -}; - -template -template -class hazard_eras::guard_ptr : public detail::guard_ptr> { - using base = detail::guard_ptr; - using Deleter = typename T::Deleter; - -public: - guard_ptr() noexcept = default; - - // Guard a marked ptr. - explicit guard_ptr(const MarkedPtr& p); - guard_ptr(const guard_ptr& p); - guard_ptr(guard_ptr&& p) noexcept; - - guard_ptr& operator=(const guard_ptr& p) noexcept; - guard_ptr& operator=(guard_ptr&& p) noexcept; - - // Atomically take snapshot of p, and *if* it points to unreclaimed object, acquire shared ownership of it. - void acquire(const concurrent_ptr& p, std::memory_order order = std::memory_order_seq_cst); - - // Like acquire, but quit early if a snapshot != expected. - bool acquire_if_equal(const concurrent_ptr& p, - const MarkedPtr& expected, - std::memory_order order = std::memory_order_seq_cst); - - // Release ownership. Postcondition: get() == nullptr. - void reset() noexcept; - - // Reset. Deleter d will be applied some time after all owners release their ownership. - void reclaim(Deleter d = Deleter()) noexcept; - -private: - using enable_concurrent_ptr = hazard_eras::enable_concurrent_ptr; - - friend base; - void do_swap(guard_ptr& g) noexcept; - - typename thread_control_block::hazard_era* he = nullptr; -}; - -namespace detail { - struct deletable_object_with_eras { - virtual void delete_self() = 0; - deletable_object_with_eras* next = nullptr; - - protected: - virtual ~deletable_object_with_eras() = default; - using era_t = size_t; - era_t construction_era{}; - era_t retirement_era{}; - template - friend class hazard_eras; - -#ifdef __clang__ - #pragma clang diagnostic push - // clang does not support dependent nested types for friend class declaration - #pragma clang diagnostic ignored "-Wunsupported-friend" -#endif - template - friend struct hazard_eras::thread_data; -#ifdef __clang__ - #pragma clang diagnostic pop -#endif - }; -} // namespace detail -} // namespace xenium::reclamation - -#define HAZARD_ERAS_IMPL -#include -#undef HAZARD_ERAS_IMPL - -#endif +// +// Copyright (c) 2018-2020 Manuel Pöter. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +#ifndef XENIUM_HAZARD_ERAS_HPP +#define XENIUM_HAZARD_ERAS_HPP + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +namespace xenium::reclamation { + +/** + * @brief This exception is thrown if a thread tries to allocate a new hazard era, but + * the number of available hazard eras is exhausted. This can only happen when + * `xenium::reclamation::he_allocation::static_strategy` is used. + */ +class bad_hazard_era_alloc : public std::runtime_error { + using std::runtime_error::runtime_error; +}; + +namespace detail { + struct deletable_object_with_eras; + + template + struct basic_he_thread_control_block; + + template class ThreadControlBlock> + struct generic_hazard_era_allocation_strategy { + static constexpr size_t K = K_; + + static size_t retired_nodes_threshold() { return A * number_of_active_hazard_eras() + B; } + + static size_t number_of_active_hazard_eras() { return number_of_active_hes.load(std::memory_order_relaxed); } + + using thread_control_block = ThreadControlBlock; + + private: + friend thread_control_block; + friend basic_he_thread_control_block; + + inline static std::atomic number_of_active_hes{0}; + }; + + template + struct static_he_thread_control_block; + + template + struct dynamic_he_thread_control_block; +} // namespace detail + +namespace he_allocation { + /** + * @brief Hazard era allocation strategy for a static number of hazard eras per thread. + * + * The threshold for the number of retired nodes is calculated as `A * K * num_threads + B`; + * `K`, `A` and `B` can be configured via template parameter. + * + * Note that in contrast to the `hazard_pointer` reclamation scheme, a `hazard_eras::guard_ptr` + * may use the same hazard era entry as a previously created `guard_ptr` if they observe the + * same era. However, since this is not guarenteed, one has to use the same upper bound for the + * number of hazard eras as when using the `hazard_pointer` scheme. + * + * @tparam K The max. number of hazard eras that can be allocated at the same time. + * @tparam A + * @tparam B + */ + template + struct static_strategy : + detail::generic_hazard_era_allocation_strategy {}; + + /** + * @brief Hazard era allocation strategy for a dynamic number of hazard eras per thread. + * + * This strategy uses a linked list of segments for the hazard eras. The first segment can + * hold `K` hazard eras; subsequently allocated segments can hold 1.5x the number of hazard + * eras as the sum of all previous segments. + * + * The threshold for the number of retired nodes is calculated as `A * available_hes + B`, where + * `available_hes` is the max. number of hazard eras that could be allocated by all threads + * at that time, without growing the number of segments. E.g., if `K = 2` and we have two threads + * each with a single segment, then `available_hes = 4`; if one thread later allocates additional + * hes and increases the number of segments such that they can hold up to 10 hazard eras, + * then `available_hes = 10+2 = 12`. + * + * `K`, `A` and `B` can be configured via template parameter. + * + * @tparam K The initial number of hazard eras (i.e., the number of hes that can be allocated + * without the need to grow). + * @tparam A + * @tparam B + */ + template + struct dynamic_strategy : + detail::generic_hazard_era_allocation_strategy {}; +} // namespace he_allocation + +template > +struct hazard_era_traits { + using allocation_strategy = AllocationStrategy; + + template + using with = hazard_era_traits>; +}; + +/** + * @brief An implementation of the hazard eras scheme proposed by Ramalhete and Correia + * \[[RC17](index.html#ref-ramalhete-2017)\]. + * + * For general information about the interface of the reclamation scheme see @ref reclamation_schemes. + * + * This class does not take a list of policies, but a `Traits` type that can be customized + * with a list of policies. The following policies are supported: + * * `xenium::policy::allocation_strategy`
+ * Defines how hazard eras are allocated and how the threshold for the number of + * retired nodes is calculated. + * Possible arguments are `xenium::reclamation::he_allocation::static_strategy` + * and 'xenium::reclamation::he_allocation::dynamic_strategy`, where both strategies + * can be further customized via their respective template parameters. + * (defaults to `xenium::reclamation::he_allocation::static_strategy<3>`) + * + * @tparam Traits + */ +template > +class hazard_eras { + using allocation_strategy = typename Traits::allocation_strategy; + using thread_control_block = typename allocation_strategy::thread_control_block; + friend detail::basic_he_thread_control_block; + + template + class guard_ptr; + +public: + /** + * @brief Customize the reclamation scheme with the given policies. + * + * The given policies are applied to the current configuration, replacing previously + * specified policies of the same type. + * + * The resulting type is the newly configured reclamation scheme. + * + * @tparam Policies list of policies to customize the behaviour + */ + template + using with = hazard_eras>; + + template > + class enable_concurrent_ptr; + + class region_guard {}; + + template + using concurrent_ptr = detail::concurrent_ptr; + + ALLOCATION_TRACKER; + +private: + struct thread_data; + + friend struct detail::deletable_object_with_eras; + + using era_t = uint64_t; + inline static std::atomic era_clock{1}; + inline static detail::thread_block_list + global_thread_block_list{}; + static thread_data& local_thread_data(); + + ALLOCATION_TRACKING_FUNCTIONS; +}; + +template +template +class hazard_eras::enable_concurrent_ptr : + public detail::deletable_object_impl, + private detail::tracked_object { +public: + static constexpr std::size_t number_of_mark_bits = N; + +protected: + enable_concurrent_ptr() noexcept { this->construction_era = era_clock.load(std::memory_order_relaxed); } + enable_concurrent_ptr(const enable_concurrent_ptr&) noexcept = default; + enable_concurrent_ptr(enable_concurrent_ptr&&) noexcept = default; + enable_concurrent_ptr& operator=(const enable_concurrent_ptr&) noexcept = default; + enable_concurrent_ptr& operator=(enable_concurrent_ptr&&) noexcept = default; + ~enable_concurrent_ptr() noexcept override = default; + +private: + friend detail::deletable_object_impl; + + template + friend class guard_ptr; +}; + +template +template +class hazard_eras::guard_ptr : public detail::guard_ptr> { + using base = detail::guard_ptr; + using Deleter = typename T::Deleter; + +public: + guard_ptr() noexcept = default; + + // Guard a marked ptr. + explicit guard_ptr(const MarkedPtr& p); + guard_ptr(const guard_ptr& p); + guard_ptr(guard_ptr&& p) noexcept; + + guard_ptr& operator=(const guard_ptr& p) noexcept; + guard_ptr& operator=(guard_ptr&& p) noexcept; + + // Atomically take snapshot of p, and *if* it points to unreclaimed object, acquire shared ownership of it. + void acquire(const concurrent_ptr& p, std::memory_order order = std::memory_order_seq_cst); + + // Like acquire, but quit early if a snapshot != expected. + bool acquire_if_equal(const concurrent_ptr& p, + const MarkedPtr& expected, + std::memory_order order = std::memory_order_seq_cst); + + // Release ownership. Postcondition: get() == nullptr. + void reset() noexcept; + + // Reset. Deleter d will be applied some time after all owners release their ownership. + void reclaim(Deleter d = Deleter()) noexcept; + +private: + using enable_concurrent_ptr = hazard_eras::enable_concurrent_ptr; + + friend base; + void do_swap(guard_ptr& g) noexcept; + + typename thread_control_block::hazard_era* he = nullptr; +}; + +namespace detail { + struct deletable_object_with_eras { + virtual void delete_self() = 0; + deletable_object_with_eras* next = nullptr; + + protected: + virtual ~deletable_object_with_eras() = default; + using era_t = size_t; + era_t construction_era{}; + era_t retirement_era{}; + template + friend class hazard_eras; + +#ifdef __clang__ + #pragma clang diagnostic push + // clang does not support dependent nested types for friend class declaration + #pragma clang diagnostic ignored "-Wunsupported-friend" +#endif + template + friend struct hazard_eras::thread_data; +#ifdef __clang__ + #pragma clang diagnostic pop +#endif + }; +} // namespace detail +} // namespace xenium::reclamation + +#define HAZARD_ERAS_IMPL +#include +#undef HAZARD_ERAS_IMPL + +#endif diff --git a/xenium/reclamation/hazard_pointer.hpp b/include/xenium/reclamation/hazard_pointer.hpp similarity index 97% rename from xenium/reclamation/hazard_pointer.hpp rename to include/xenium/reclamation/hazard_pointer.hpp index 1c39ef9..b49397b 100644 --- a/xenium/reclamation/hazard_pointer.hpp +++ b/include/xenium/reclamation/hazard_pointer.hpp @@ -1,237 +1,237 @@ -// -// Copyright (c) 2018-2020 Manuel Pöter. -// Licensed under the MIT License. See LICENSE file in the project root for full license information. -// - -#ifndef XENIUM_HAZARD_POINTER_HPP -#define XENIUM_HAZARD_POINTER_HPP - -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include - -namespace xenium::reclamation { - -/** - * @brief This exception is thrown if a thread tries to allocate a new hazard pointer, but - * the number of available hazard pointers is exhausted. This can only happen when - * `xenium::reclamation::hp_allocation::static_strategy` is used. - */ -class bad_hazard_pointer_alloc : public std::runtime_error { - using std::runtime_error::runtime_error; -}; - -namespace detail { - template - struct basic_hp_thread_control_block; - - template class ThreadControlBlock> - struct generic_hp_allocation_strategy { - static constexpr size_t K = K_; - - static size_t retired_nodes_threshold() { return A * number_of_active_hazard_pointers() + B; } - - static size_t number_of_active_hazard_pointers() { return number_of_active_hps.load(std::memory_order_relaxed); } - - using thread_control_block = ThreadControlBlock; - - private: - friend thread_control_block; - friend basic_hp_thread_control_block; - - inline static std::atomic number_of_active_hps{0}; - }; - - template - struct static_hp_thread_control_block; - - template - struct dynamic_hp_thread_control_block; -} // namespace detail - -namespace hp_allocation { - /** - * @brief Hazard pointer allocation strategy for a static number of hazard pointers per thread. - * - * The threshold for the number of retired nodes is calculated as `A * K * num_threads + B`; - * `K`, `A` and `B` can be configured via template parameter. - * - * @tparam K The max. number of hazard pointers that can be allocated at the same time. - * @tparam A - * @tparam B - */ - template - struct static_strategy : detail::generic_hp_allocation_strategy {}; - - /** - * @brief Hazard pointer allocation strategy for a dynamic number of hazard pointers per thread. - * - * This strategy uses a linked list of segments for the hazard pointers. The first segment can - * hold `K` hazard pointers; subsequently allocated segments can hold 1.5x the number of hazard - * pointers as the sum of all previous segments. - * - * The threshold for the number of retired nodes is calculated as `A * available_hps + B`, where - * `available_hps` is the max. number of hazard pointers that could be allocated by all threads - * at that time, without growing the number of segments. E.g., if `K = 2` and we have two threads - * each with a single segment, then `available_hps = 4`; if one thread later allocates additional - * hps and increases the number of segments such that they can hold up to 10 hazard pointers, - * then `available_hps = 10+2 = 12`. - * - * `K`, `A` and `B` can be configured via template parameter. - * - * @tparam K The initial number of hazard pointers (i.e., the number of hps that can be allocated - * without the need to grow). - * @tparam A - * @tparam B - */ - template - struct dynamic_strategy : detail::generic_hp_allocation_strategy {}; -} // namespace hp_allocation - -template > -struct hazard_pointer_traits { - using allocation_strategy = AllocationStrategy; - - template - using with = - hazard_pointer_traits>; -}; - -/** - * @brief An implementation of the hazard pointers reclamation scheme as proposed by Michael - * \[[Mic04](index.html#ref-michael-2004)\]. - * - * For general information about the interface of the reclamation scheme see @ref reclamation_schemes. - * - * This class does not take a list of policies, but a `Traits` type that can be customized - * with a list of policies. The following policies are supported: - * * `xenium::policy::allocation_strategy`
- * Defines how hazard pointers are allocated and how the threshold for the number of - * retired nodes is calculated. - * Possible arguments are `xenium::reclamation::hp_allocation::static_strategy` - * and 'xenium::reclamation::hp_allocation::dynamic_strategy`, where both strategies - * can be further customized via their respective template parameters. - * (defaults to `xenium::reclamation::he_allocation::static_strategy<3>`) - * - * @tparam Traits - */ -template > -class hazard_pointer { - using allocation_strategy = typename Traits::allocation_strategy; - using thread_control_block = typename allocation_strategy::thread_control_block; - friend detail::basic_hp_thread_control_block; - - template - class guard_ptr; - -public: - /** - * @brief Customize the reclamation scheme with the given policies. - * - * The given policies are applied to the current configuration, replacing previously - * specified policies of the same type. - * - * The resulting type is the newly configured reclamation scheme. - * - * @tparam Policies list of policies to customize the behaviour - */ - template - using with = hazard_pointer>; - - template > - class enable_concurrent_ptr; - - class region_guard {}; - - template - using concurrent_ptr = detail::concurrent_ptr; - - ALLOCATION_TRACKER; - -private: - struct thread_data; - - inline static detail::thread_block_list global_thread_block_list; - inline static thread_local thread_data local_thread_data; - - ALLOCATION_TRACKING_FUNCTIONS; -}; - -template -template -class hazard_pointer::enable_concurrent_ptr : - private detail::deletable_object_impl, - private detail::tracked_object { -public: - static constexpr std::size_t number_of_mark_bits = N; - -protected: - enable_concurrent_ptr() noexcept = default; - enable_concurrent_ptr(const enable_concurrent_ptr&) noexcept = default; - enable_concurrent_ptr(enable_concurrent_ptr&&) noexcept = default; - enable_concurrent_ptr& operator=(const enable_concurrent_ptr&) noexcept = default; - enable_concurrent_ptr& operator=(enable_concurrent_ptr&&) noexcept = default; - ~enable_concurrent_ptr() noexcept override = default; - -private: - friend detail::deletable_object_impl; - - template - friend class guard_ptr; -}; - -template -template -class hazard_pointer::guard_ptr : public detail::guard_ptr> { - using base = detail::guard_ptr; - using Deleter = typename T::Deleter; - -public: - guard_ptr() noexcept = default; - - // Guard a marked ptr. - explicit guard_ptr(const MarkedPtr& p); - guard_ptr(const guard_ptr& p); - guard_ptr(guard_ptr&& p) noexcept; - - guard_ptr& operator=(const guard_ptr& p); - guard_ptr& operator=(guard_ptr&& p) noexcept; - - // Atomically take snapshot of p, and *if* it points to unreclaimed object, acquire shared ownership of it. - void acquire(const concurrent_ptr& p, std::memory_order order = std::memory_order_seq_cst); - - // Like acquire, but quit early if a snapshot != expected. - bool acquire_if_equal(const concurrent_ptr& p, - const MarkedPtr& expected, - std::memory_order order = std::memory_order_seq_cst); - - // Release ownership. Postcondition: get() == nullptr. - void reset() noexcept; - - // Reset. Deleter d will be applied some time after all owners release their ownership. - void reclaim(Deleter d = Deleter()) noexcept; - -private: - using enable_concurrent_ptr = hazard_pointer::enable_concurrent_ptr; - - friend base; - void do_swap(guard_ptr& g) noexcept; - - typename thread_control_block::hazard_pointer* hp = nullptr; -}; -} // namespace xenium::reclamation - -#define HAZARD_POINTER_IMPL -#include -#undef HAZARD_POINTER_IMPL - -#endif +// +// Copyright (c) 2018-2020 Manuel Pöter. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +#ifndef XENIUM_HAZARD_POINTER_HPP +#define XENIUM_HAZARD_POINTER_HPP + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +namespace xenium::reclamation { + +/** + * @brief This exception is thrown if a thread tries to allocate a new hazard pointer, but + * the number of available hazard pointers is exhausted. This can only happen when + * `xenium::reclamation::hp_allocation::static_strategy` is used. + */ +class bad_hazard_pointer_alloc : public std::runtime_error { + using std::runtime_error::runtime_error; +}; + +namespace detail { + template + struct basic_hp_thread_control_block; + + template class ThreadControlBlock> + struct generic_hp_allocation_strategy { + static constexpr size_t K = K_; + + static size_t retired_nodes_threshold() { return A * number_of_active_hazard_pointers() + B; } + + static size_t number_of_active_hazard_pointers() { return number_of_active_hps.load(std::memory_order_relaxed); } + + using thread_control_block = ThreadControlBlock; + + private: + friend thread_control_block; + friend basic_hp_thread_control_block; + + inline static std::atomic number_of_active_hps{0}; + }; + + template + struct static_hp_thread_control_block; + + template + struct dynamic_hp_thread_control_block; +} // namespace detail + +namespace hp_allocation { + /** + * @brief Hazard pointer allocation strategy for a static number of hazard pointers per thread. + * + * The threshold for the number of retired nodes is calculated as `A * K * num_threads + B`; + * `K`, `A` and `B` can be configured via template parameter. + * + * @tparam K The max. number of hazard pointers that can be allocated at the same time. + * @tparam A + * @tparam B + */ + template + struct static_strategy : detail::generic_hp_allocation_strategy {}; + + /** + * @brief Hazard pointer allocation strategy for a dynamic number of hazard pointers per thread. + * + * This strategy uses a linked list of segments for the hazard pointers. The first segment can + * hold `K` hazard pointers; subsequently allocated segments can hold 1.5x the number of hazard + * pointers as the sum of all previous segments. + * + * The threshold for the number of retired nodes is calculated as `A * available_hps + B`, where + * `available_hps` is the max. number of hazard pointers that could be allocated by all threads + * at that time, without growing the number of segments. E.g., if `K = 2` and we have two threads + * each with a single segment, then `available_hps = 4`; if one thread later allocates additional + * hps and increases the number of segments such that they can hold up to 10 hazard pointers, + * then `available_hps = 10+2 = 12`. + * + * `K`, `A` and `B` can be configured via template parameter. + * + * @tparam K The initial number of hazard pointers (i.e., the number of hps that can be allocated + * without the need to grow). + * @tparam A + * @tparam B + */ + template + struct dynamic_strategy : detail::generic_hp_allocation_strategy {}; +} // namespace hp_allocation + +template > +struct hazard_pointer_traits { + using allocation_strategy = AllocationStrategy; + + template + using with = + hazard_pointer_traits>; +}; + +/** + * @brief An implementation of the hazard pointers reclamation scheme as proposed by Michael + * \[[Mic04](index.html#ref-michael-2004)\]. + * + * For general information about the interface of the reclamation scheme see @ref reclamation_schemes. + * + * This class does not take a list of policies, but a `Traits` type that can be customized + * with a list of policies. The following policies are supported: + * * `xenium::policy::allocation_strategy`
+ * Defines how hazard pointers are allocated and how the threshold for the number of + * retired nodes is calculated. + * Possible arguments are `xenium::reclamation::hp_allocation::static_strategy` + * and 'xenium::reclamation::hp_allocation::dynamic_strategy`, where both strategies + * can be further customized via their respective template parameters. + * (defaults to `xenium::reclamation::he_allocation::static_strategy<3>`) + * + * @tparam Traits + */ +template > +class hazard_pointer { + using allocation_strategy = typename Traits::allocation_strategy; + using thread_control_block = typename allocation_strategy::thread_control_block; + friend detail::basic_hp_thread_control_block; + + template + class guard_ptr; + +public: + /** + * @brief Customize the reclamation scheme with the given policies. + * + * The given policies are applied to the current configuration, replacing previously + * specified policies of the same type. + * + * The resulting type is the newly configured reclamation scheme. + * + * @tparam Policies list of policies to customize the behaviour + */ + template + using with = hazard_pointer>; + + template > + class enable_concurrent_ptr; + + class region_guard {}; + + template + using concurrent_ptr = detail::concurrent_ptr; + + ALLOCATION_TRACKER; + +private: + struct thread_data; + + inline static detail::thread_block_list global_thread_block_list; + inline static thread_local thread_data local_thread_data; + + ALLOCATION_TRACKING_FUNCTIONS; +}; + +template +template +class hazard_pointer::enable_concurrent_ptr : + private detail::deletable_object_impl, + private detail::tracked_object { +public: + static constexpr std::size_t number_of_mark_bits = N; + +protected: + enable_concurrent_ptr() noexcept = default; + enable_concurrent_ptr(const enable_concurrent_ptr&) noexcept = default; + enable_concurrent_ptr(enable_concurrent_ptr&&) noexcept = default; + enable_concurrent_ptr& operator=(const enable_concurrent_ptr&) noexcept = default; + enable_concurrent_ptr& operator=(enable_concurrent_ptr&&) noexcept = default; + ~enable_concurrent_ptr() noexcept override = default; + +private: + friend detail::deletable_object_impl; + + template + friend class guard_ptr; +}; + +template +template +class hazard_pointer::guard_ptr : public detail::guard_ptr> { + using base = detail::guard_ptr; + using Deleter = typename T::Deleter; + +public: + guard_ptr() noexcept = default; + + // Guard a marked ptr. + explicit guard_ptr(const MarkedPtr& p); + guard_ptr(const guard_ptr& p); + guard_ptr(guard_ptr&& p) noexcept; + + guard_ptr& operator=(const guard_ptr& p); + guard_ptr& operator=(guard_ptr&& p) noexcept; + + // Atomically take snapshot of p, and *if* it points to unreclaimed object, acquire shared ownership of it. + void acquire(const concurrent_ptr& p, std::memory_order order = std::memory_order_seq_cst); + + // Like acquire, but quit early if a snapshot != expected. + bool acquire_if_equal(const concurrent_ptr& p, + const MarkedPtr& expected, + std::memory_order order = std::memory_order_seq_cst); + + // Release ownership. Postcondition: get() == nullptr. + void reset() noexcept; + + // Reset. Deleter d will be applied some time after all owners release their ownership. + void reclaim(Deleter d = Deleter()) noexcept; + +private: + using enable_concurrent_ptr = hazard_pointer::enable_concurrent_ptr; + + friend base; + void do_swap(guard_ptr& g) noexcept; + + typename thread_control_block::hazard_pointer* hp = nullptr; +}; +} // namespace xenium::reclamation + +#define HAZARD_POINTER_IMPL +#include +#undef HAZARD_POINTER_IMPL + +#endif diff --git a/xenium/reclamation/impl/generic_epoch_based.hpp b/include/xenium/reclamation/impl/generic_epoch_based.hpp similarity index 100% rename from xenium/reclamation/impl/generic_epoch_based.hpp rename to include/xenium/reclamation/impl/generic_epoch_based.hpp diff --git a/xenium/reclamation/impl/hazard_eras.hpp b/include/xenium/reclamation/impl/hazard_eras.hpp similarity index 97% rename from xenium/reclamation/impl/hazard_eras.hpp rename to include/xenium/reclamation/impl/hazard_eras.hpp index 3e0587d..d40fd7d 100644 --- a/xenium/reclamation/impl/hazard_eras.hpp +++ b/include/xenium/reclamation/impl/hazard_eras.hpp @@ -1,554 +1,554 @@ -// -// Copyright (c) 2018-2020 Manuel Pöter. -// Licensed under the MIT License. See LICENSE file in the project root for full license information. -// - -#ifndef HAZARD_ERAS_IMPL - #error "This is an impl file and must not be included directly!" -#endif - -#include -#include - -#include -#include -#include - -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4324) // structure was padded due to alignment specifier - #pragma warning(disable : 26495) // uninitialized member variable -#endif - -namespace xenium::reclamation { - -template -template -hazard_eras::guard_ptr::guard_ptr(const MarkedPtr& p) : base(p), he() { - if (this->ptr.get() != nullptr) { - auto era = era_clock.load(std::memory_order_relaxed); - he = local_thread_data().alloc_hazard_era(era); - } -} - -template -template -hazard_eras::guard_ptr::guard_ptr(const guard_ptr& p) : base(p.ptr), he(p.he) { - if (he) { - he->add_guard(); - } -} - -template -template -hazard_eras::guard_ptr::guard_ptr(guard_ptr&& p) noexcept : base(p.ptr), he(p.he) { - p.ptr.reset(); - p.he = nullptr; -} - -template -template -auto hazard_eras::guard_ptr::operator=(const guard_ptr& p) noexcept -> guard_ptr& { - if (&p == this) { - return *this; - } - - reset(); - this->ptr = p.ptr; - he = p.he; - if (he) { - he->add_guard(); - } - return *this; -} - -template -template -auto hazard_eras::guard_ptr::operator=(guard_ptr&& p) noexcept -> guard_ptr& { - if (&p == this) { - return *this; - } - - reset(); - this->ptr = std::move(p.ptr); - this->he = p.he; - p.ptr.reset(); - p.he = nullptr; - return *this; -} - -template -template -void hazard_eras::guard_ptr::acquire(const concurrent_ptr& p, std::memory_order order) { - if (order == std::memory_order_relaxed || order == std::memory_order_consume) { - // we have to use memory_order_acquire (or something stricter) to ensure that - // the era_clock.load cannot return an outdated value. - order = std::memory_order_acquire; - } - era_t prev_era = he == nullptr ? 0 : he->get_era(); - for (;;) { - // (1) - this load operation synchronizes-with any release operation on p. - // we have to use acquire here to ensure that the subsequent era_clock.load - // sees a value >= p.construction_era - auto value = p.load(order); - - auto era = era_clock.load(std::memory_order_relaxed); - if (era == prev_era) { - this->ptr = value; - return; - } - - if (he != nullptr) { - assert(he->get_era() != era); - if (he->guards() == 1) { - // we are the only guard using this HE instance -> reuse it and simply update the era - he->set_era(era); - prev_era = era; - continue; - } - he->release_guard(); - he = nullptr; - } - assert(he == nullptr); - he = local_thread_data().alloc_hazard_era(era); - prev_era = era; - } -} - -template -template -bool hazard_eras::guard_ptr::acquire_if_equal(const concurrent_ptr& p, - const MarkedPtr& expected, - std::memory_order order) { - if (order == std::memory_order_relaxed || order == std::memory_order_consume) { - // we have to use memory_order_acquire (or something stricter) to ensure that - // the era_clock.load cannot return an outdated value. - order = std::memory_order_acquire; - } - - // (2) - this load operation synchronizes-with any release operation on p. - // we have to use acquire here to ensure that the subsequent era_clock.load - // sees a value >= p.construction_era - auto p1 = p.load(order); - if (p1 == nullptr || p1 != expected) { - reset(); - return p1 == expected; - } - - const auto era = era_clock.load(std::memory_order_relaxed); - if (he != nullptr && he->guards() == 1) { - he->set_era(era); - } else { - if (he != nullptr) { - he->release_guard(); - } - - he = local_thread_data().alloc_hazard_era(era); - } - - this->ptr = p.load(std::memory_order_relaxed); - if (this->ptr != p1) { - reset(); - return false; - } - return true; -} - -template -template -void hazard_eras::guard_ptr::reset() noexcept { - local_thread_data().release_hazard_era(he); - assert(this->he == nullptr); - this->ptr.reset(); -} - -template -template -void hazard_eras::guard_ptr::do_swap(guard_ptr& g) noexcept { - std::swap(he, g.he); -} - -template -template -void hazard_eras::guard_ptr::reclaim(Deleter d) noexcept { - auto* p = this->ptr.get(); - reset(); - p->set_deleter(std::move(d)); - // (3) - this release fetch-add synchronizes-with the seq-cst fence (5) - p->retirement_era = era_clock.fetch_add(1, std::memory_order_release); - - if (local_thread_data().add_retired_node(p) >= allocation_strategy::retired_nodes_threshold()) { - local_thread_data().scan(); - } -} - -namespace detail { - template - struct alignas(64) basic_he_thread_control_block : - detail::thread_block_list::entry, - aligned_object> { - using era_t = uint64_t; - - ~basic_he_thread_control_block() { assert(last_hazard_era != nullptr); } - - struct hazard_era { - void set_era(era_t era) { - assert(era != 0); - // (4) - this release-store synchronizes-with the acquire-fence (10) - value.store(marked_ptr(reinterpret_cast(era << 1)), std::memory_order_release); - // This release is required because when acquire/acquire_if_equal is called on a - // guard_ptr with with an active HE entry, set_era is called without an intermediate - // call to set_link, i.e., the protected era is updated. This ensures the required - // happens-before relation between releasing a guard_ptr to a node and reclamation - // of that node. - - // (5) - this seq_cst-fence enforces a total order with the seq_cst-fence (9) - // and synchronizes-with the release-fetch-add (3) - XENIUM_THREAD_FENCE(std::memory_order_seq_cst); - } - - [[nodiscard]] era_t get_era() const { - era_t result = 0; - bool success = try_get_era(result); - (void)success; - assert(success); - assert(result != 0); - return result; - } - - [[nodiscard]] uint64_t guards() const { return guard_cnt; } - uint64_t add_guard() { return ++guard_cnt; } - uint64_t release_guard() { return --guard_cnt; } - - bool try_get_era(era_t& result) const { - // TSan does not support explicit fences, so we cannot rely on the acquire-fence (10) - // but have to perform an acquire-load here to avoid false positives. - constexpr auto memory_order = TSAN_MEMORY_ORDER(std::memory_order_acquire, std::memory_order_relaxed); - auto v = value.load(memory_order); - if (v.mark() == 0) { - result = reinterpret_cast(v.get()) >> 1; - return true; - } - return false; // value contains a link - } - - void set_link(hazard_era* link) { - assert(guard_cnt == 0); - // (6) - this release store synchronizes-with the acquire fence (10) - value.store(marked_ptr(reinterpret_cast(link), 1), std::memory_order_release); - } - [[nodiscard]] hazard_era* get_link() const { - assert(is_link()); - return reinterpret_cast(value.load(std::memory_order_relaxed).get()); - } - - [[nodiscard]] bool is_link() const { return value.load(std::memory_order_relaxed).mark() != 0; } - - private: - using marked_ptr = xenium::marked_ptr; - // since we use the hazard era array to build our internal linked list of hazard eras we set - // the LSB to signal that this is an internal pointer and not a pointer to a protected object. - std::atomic value{nullptr}; - // the number of guard_ptrs that reference this instance and therefore observe the same era - uint64_t guard_cnt = 0; - }; - - using hint = hazard_era*; - - void initialize(hint& hint) { - Strategy::number_of_active_hes.fetch_add(self().number_of_hes(), std::memory_order_relaxed); - hint = initialize_block(self()); - } - - void abandon() { - Strategy::number_of_active_hes.fetch_sub(self().number_of_hes(), std::memory_order_relaxed); - detail::thread_block_list::entry::abandon(); - } - - hazard_era* alloc_hazard_era(hint& hint, era_t era) { - if (last_hazard_era && last_era == era) { - last_hazard_era->add_guard(); - return last_hazard_era; - } - auto result = hint; - if (result == nullptr) { - result = self().need_more_hes(); - } - - hint = result->get_link(); - result->set_era(era); - result->add_guard(); - - last_hazard_era = result; - last_era = era; - return result; - } - - void release_hazard_era(hazard_era*& he, hint& hint) { - assert(he != nullptr); - if (he->release_guard() == 0) { - if (he == last_hazard_era) { - last_hazard_era = nullptr; - } - - he->set_link(hint); - hint = he; - } - he = nullptr; - } - - protected: - Derived& self() { return static_cast(*this); } - - hazard_era* begin() { return &eras[0]; } - hazard_era* end() { return &eras[Strategy::K]; } - [[nodiscard]] const hazard_era* begin() const { return &eras[0]; } - [[nodiscard]] const hazard_era* end() const { return &eras[Strategy::K]; } - - template - static hazard_era* initialize_block(T& block) { - auto begin = block.begin(); - auto end = block.end() - 1; // the last element is handled specially, so loop only over n-1 entries - for (auto it = begin; it != end;) { - auto next = it + 1; - it->set_link(next); - it = next; - } - end->set_link(block.initialize_next_block()); - return begin; - } - - static void - gather_protected_eras(std::vector& protected_eras, const hazard_era* begin, const hazard_era* end) { - for (auto it = begin; it != end; ++it) { - era_t era; - if (it->try_get_era(era)) { - protected_eras.push_back(era); - } - } - } - - hazard_era* last_hazard_era; - era_t last_era; - hazard_era eras[Strategy::K]; - }; - - template - struct static_he_thread_control_block : - basic_he_thread_control_block> { - using base = basic_he_thread_control_block; - using hazard_era = typename base::hazard_era; - using era_t = typename base::era_t; - friend base; - - void gather_protected_eras(std::vector& protected_eras) const { - base::gather_protected_eras(protected_eras, this->begin(), this->end()); - } - - private: - hazard_era* need_more_hes() { throw bad_hazard_era_alloc("hazard era pool exceeded"); } - [[nodiscard]] constexpr size_t number_of_hes() const { return Strategy::K; } - [[nodiscard]] constexpr hazard_era* initialize_next_block() const { return nullptr; } - }; - - template - struct dynamic_he_thread_control_block : - basic_he_thread_control_block> { - using base = basic_he_thread_control_block; - using hazard_era = typename base::hazard_era; - using era_t = typename base::era_t; - friend base; - - void gather_protected_eras(std::vector& protected_eras) const { - gather_protected_eras(*this, protected_eras); - } - - private: - struct alignas(64) hazard_eras_block : aligned_object { - explicit hazard_eras_block(size_t size) : size(size) { - for (auto it = begin(); it != end(); ++it) { - new (it) hazard_era; - } - } - - hazard_era* begin() { return reinterpret_cast(this + 1); } - hazard_era* end() { return begin() + size; } - - [[nodiscard]] const hazard_era* begin() const { return reinterpret_cast(this + 1); } - [[nodiscard]] const hazard_era* end() const { return begin() + size; } - - [[nodiscard]] const hazard_eras_block* next_block() const { return next; } - hazard_era* initialize_next_block() { return next ? base::initialize_block(*next) : nullptr; } - - hazard_eras_block* next = nullptr; - const size_t size; - }; - - [[nodiscard]] const hazard_eras_block* next_block() const { - // (7) - this acquire-load synchronizes-with the release-store (8) - return he_block.load(std::memory_order_acquire); - } - [[nodiscard]] size_t number_of_hes() const { return total_number_of_hes; } - hazard_era* need_more_hes() { return allocate_new_hazard_eras_block(); } - - hazard_era* initialize_next_block() { - auto block = he_block.load(std::memory_order_relaxed); - return block ? base::initialize_block(*block) : nullptr; - } - - template - static void gather_protected_eras(const T& block, std::vector& protected_eras) { - base::gather_protected_eras(protected_eras, block.begin(), block.end()); - - const auto* next = block.next_block(); - if (next) { - gather_protected_eras(*next, protected_eras); - } - } - - hazard_era* allocate_new_hazard_eras_block() { - size_t hes = std::max(static_cast(Strategy::K), total_number_of_hes / 2); - total_number_of_hes += hes; - Strategy::number_of_active_hes.fetch_add(hes, std::memory_order_relaxed); - - size_t buffer_size = sizeof(hazard_eras_block) + hes * sizeof(hazard_era); - void* buffer = hazard_eras_block::operator new(buffer_size); - auto block = ::new (buffer) hazard_eras_block(hes); - auto result = this->initialize_block(*block); - block->next = he_block.load(std::memory_order_relaxed); - // (8) - this release-store synchronizes-with the acquire-load (7) - he_block.store(block, std::memory_order_release); - return result; - } - - size_t total_number_of_hes = Strategy::K; - std::atomic he_block; - }; -} // namespace detail - -template -struct alignas(64) hazard_eras::thread_data : aligned_object { - using HE = typename thread_control_block::hazard_era*; - - ~thread_data() { - if (retire_list != nullptr) { - scan(); - if (retire_list != nullptr) { - global_thread_block_list.abandon_retired_nodes(retire_list); - } - retire_list = nullptr; - } - - if (control_block != nullptr) { - global_thread_block_list.release_entry(control_block); - control_block = nullptr; - } - } - - HE alloc_hazard_era(era_t era) { - ensure_has_control_block(); - return control_block->alloc_hazard_era(hint, era); - } - - void release_hazard_era(HE& he) { - if (he) { - assert(control_block != nullptr); - control_block->release_hazard_era(he, hint); - } - } - - std::size_t add_retired_node(detail::deletable_object_with_eras* p) { - p->next = retire_list; - retire_list = p; - return ++number_of_retired_nodes; - } - - void scan() { - std::vector protected_eras; - protected_eras.reserve(allocation_strategy::number_of_active_hazard_eras()); - - // (9) - this seq_cst-fence enforces a total order with the seq_cst-fence (5) - XENIUM_THREAD_FENCE(std::memory_order_seq_cst); - - auto adopted_nodes = global_thread_block_list.adopt_abandoned_retired_nodes(); - - std::for_each( - global_thread_block_list.begin(), global_thread_block_list.end(), [&protected_eras](const auto& entry) { - // TSan does not support explicit fences, so we cannot rely on the acquire-fence (10) - // but have to perform an acquire-load here to avoid false positives. - constexpr auto memory_order = TSAN_MEMORY_ORDER(std::memory_order_acquire, std::memory_order_relaxed); - if (entry.is_active(memory_order)) { - entry.gather_protected_eras(protected_eras); - } - }); - - // (10) - this acquire-fence synchronizes-with the release-store (4, 6) - XENIUM_THREAD_FENCE(std::memory_order_acquire); - - std::sort(protected_eras.begin(), protected_eras.end()); - auto last = std::unique(protected_eras.begin(), protected_eras.end()); - protected_eras.erase(last, protected_eras.end()); - - auto* list = retire_list; - retire_list = nullptr; - number_of_retired_nodes = 0; - reclaim_nodes(list, protected_eras); - reclaim_nodes(adopted_nodes, protected_eras); - } - -private: - void ensure_has_control_block() { - if (control_block == nullptr) { - control_block = global_thread_block_list.acquire_inactive_entry(); - control_block->initialize(hint); - control_block->activate(); - } - } - - void reclaim_nodes(detail::deletable_object_with_eras* list, const std::vector& protected_eras) { - while (list != nullptr) { - auto* cur = list; - list = list->next; - - auto era_it = std::lower_bound(protected_eras.begin(), protected_eras.end(), cur->construction_era); - if (era_it == protected_eras.end() || *era_it > cur->retirement_era) { - cur->delete_self(); - } else { - add_retired_node(cur); - } - } - } - - detail::deletable_object_with_eras* retire_list = nullptr; - std::size_t number_of_retired_nodes = 0; - typename thread_control_block::hint hint; - - thread_control_block* control_block = nullptr; - - friend class hazard_eras; - ALLOCATION_COUNTER(hazard_eras); -}; - -template -inline typename hazard_eras::thread_data& hazard_eras::local_thread_data() { - // workaround for a Clang-8 issue that causes multiple re-initializations of thread_local variables - static thread_local thread_data local_thread_data; - return local_thread_data; -} - -#ifdef TRACK_ALLOCATIONS -template -inline void hazard_eras::count_allocation() { - local_thread_data().allocation_counter.count_allocation(); -} - -template -inline void hazard_eras::count_reclamation() { - local_thread_data().allocation_counter.count_reclamation(); -} -#endif -} // namespace xenium::reclamation - -#ifdef _MSC_VER - #pragma warning(pop) -#endif +// +// Copyright (c) 2018-2020 Manuel Pöter. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +#ifndef HAZARD_ERAS_IMPL + #error "This is an impl file and must not be included directly!" +#endif + +#include +#include + +#include +#include +#include + +#ifdef _MSC_VER + #pragma warning(push) + #pragma warning(disable : 4324) // structure was padded due to alignment specifier + #pragma warning(disable : 26495) // uninitialized member variable +#endif + +namespace xenium::reclamation { + +template +template +hazard_eras::guard_ptr::guard_ptr(const MarkedPtr& p) : base(p), he() { + if (this->ptr.get() != nullptr) { + auto era = era_clock.load(std::memory_order_relaxed); + he = local_thread_data().alloc_hazard_era(era); + } +} + +template +template +hazard_eras::guard_ptr::guard_ptr(const guard_ptr& p) : base(p.ptr), he(p.he) { + if (he) { + he->add_guard(); + } +} + +template +template +hazard_eras::guard_ptr::guard_ptr(guard_ptr&& p) noexcept : base(p.ptr), he(p.he) { + p.ptr.reset(); + p.he = nullptr; +} + +template +template +auto hazard_eras::guard_ptr::operator=(const guard_ptr& p) noexcept -> guard_ptr& { + if (&p == this) { + return *this; + } + + reset(); + this->ptr = p.ptr; + he = p.he; + if (he) { + he->add_guard(); + } + return *this; +} + +template +template +auto hazard_eras::guard_ptr::operator=(guard_ptr&& p) noexcept -> guard_ptr& { + if (&p == this) { + return *this; + } + + reset(); + this->ptr = std::move(p.ptr); + this->he = p.he; + p.ptr.reset(); + p.he = nullptr; + return *this; +} + +template +template +void hazard_eras::guard_ptr::acquire(const concurrent_ptr& p, std::memory_order order) { + if (order == std::memory_order_relaxed || order == std::memory_order_consume) { + // we have to use memory_order_acquire (or something stricter) to ensure that + // the era_clock.load cannot return an outdated value. + order = std::memory_order_acquire; + } + era_t prev_era = he == nullptr ? 0 : he->get_era(); + for (;;) { + // (1) - this load operation synchronizes-with any release operation on p. + // we have to use acquire here to ensure that the subsequent era_clock.load + // sees a value >= p.construction_era + auto value = p.load(order); + + auto era = era_clock.load(std::memory_order_relaxed); + if (era == prev_era) { + this->ptr = value; + return; + } + + if (he != nullptr) { + assert(he->get_era() != era); + if (he->guards() == 1) { + // we are the only guard using this HE instance -> reuse it and simply update the era + he->set_era(era); + prev_era = era; + continue; + } + he->release_guard(); + he = nullptr; + } + assert(he == nullptr); + he = local_thread_data().alloc_hazard_era(era); + prev_era = era; + } +} + +template +template +bool hazard_eras::guard_ptr::acquire_if_equal(const concurrent_ptr& p, + const MarkedPtr& expected, + std::memory_order order) { + if (order == std::memory_order_relaxed || order == std::memory_order_consume) { + // we have to use memory_order_acquire (or something stricter) to ensure that + // the era_clock.load cannot return an outdated value. + order = std::memory_order_acquire; + } + + // (2) - this load operation synchronizes-with any release operation on p. + // we have to use acquire here to ensure that the subsequent era_clock.load + // sees a value >= p.construction_era + auto p1 = p.load(order); + if (p1 == nullptr || p1 != expected) { + reset(); + return p1 == expected; + } + + const auto era = era_clock.load(std::memory_order_relaxed); + if (he != nullptr && he->guards() == 1) { + he->set_era(era); + } else { + if (he != nullptr) { + he->release_guard(); + } + + he = local_thread_data().alloc_hazard_era(era); + } + + this->ptr = p.load(std::memory_order_relaxed); + if (this->ptr != p1) { + reset(); + return false; + } + return true; +} + +template +template +void hazard_eras::guard_ptr::reset() noexcept { + local_thread_data().release_hazard_era(he); + assert(this->he == nullptr); + this->ptr.reset(); +} + +template +template +void hazard_eras::guard_ptr::do_swap(guard_ptr& g) noexcept { + std::swap(he, g.he); +} + +template +template +void hazard_eras::guard_ptr::reclaim(Deleter d) noexcept { + auto* p = this->ptr.get(); + reset(); + p->set_deleter(std::move(d)); + // (3) - this release fetch-add synchronizes-with the seq-cst fence (5) + p->retirement_era = era_clock.fetch_add(1, std::memory_order_release); + + if (local_thread_data().add_retired_node(p) >= allocation_strategy::retired_nodes_threshold()) { + local_thread_data().scan(); + } +} + +namespace detail { + template + struct alignas(64) basic_he_thread_control_block : + detail::thread_block_list::entry, + aligned_object> { + using era_t = uint64_t; + + ~basic_he_thread_control_block() { assert(last_hazard_era != nullptr); } + + struct hazard_era { + void set_era(era_t era) { + assert(era != 0); + // (4) - this release-store synchronizes-with the acquire-fence (10) + value.store(marked_ptr(reinterpret_cast(era << 1)), std::memory_order_release); + // This release is required because when acquire/acquire_if_equal is called on a + // guard_ptr with with an active HE entry, set_era is called without an intermediate + // call to set_link, i.e., the protected era is updated. This ensures the required + // happens-before relation between releasing a guard_ptr to a node and reclamation + // of that node. + + // (5) - this seq_cst-fence enforces a total order with the seq_cst-fence (9) + // and synchronizes-with the release-fetch-add (3) + XENIUM_THREAD_FENCE(std::memory_order_seq_cst); + } + + [[nodiscard]] era_t get_era() const { + era_t result = 0; + bool success = try_get_era(result); + (void)success; + assert(success); + assert(result != 0); + return result; + } + + [[nodiscard]] uint64_t guards() const { return guard_cnt; } + uint64_t add_guard() { return ++guard_cnt; } + uint64_t release_guard() { return --guard_cnt; } + + bool try_get_era(era_t& result) const { + // TSan does not support explicit fences, so we cannot rely on the acquire-fence (10) + // but have to perform an acquire-load here to avoid false positives. + constexpr auto memory_order = TSAN_MEMORY_ORDER(std::memory_order_acquire, std::memory_order_relaxed); + auto v = value.load(memory_order); + if (v.mark() == 0) { + result = reinterpret_cast(v.get()) >> 1; + return true; + } + return false; // value contains a link + } + + void set_link(hazard_era* link) { + assert(guard_cnt == 0); + // (6) - this release store synchronizes-with the acquire fence (10) + value.store(marked_ptr(reinterpret_cast(link), 1), std::memory_order_release); + } + [[nodiscard]] hazard_era* get_link() const { + assert(is_link()); + return reinterpret_cast(value.load(std::memory_order_relaxed).get()); + } + + [[nodiscard]] bool is_link() const { return value.load(std::memory_order_relaxed).mark() != 0; } + + private: + using marked_ptr = xenium::marked_ptr; + // since we use the hazard era array to build our internal linked list of hazard eras we set + // the LSB to signal that this is an internal pointer and not a pointer to a protected object. + std::atomic value{nullptr}; + // the number of guard_ptrs that reference this instance and therefore observe the same era + uint64_t guard_cnt = 0; + }; + + using hint = hazard_era*; + + void initialize(hint& hint) { + Strategy::number_of_active_hes.fetch_add(self().number_of_hes(), std::memory_order_relaxed); + hint = initialize_block(self()); + } + + void abandon() { + Strategy::number_of_active_hes.fetch_sub(self().number_of_hes(), std::memory_order_relaxed); + detail::thread_block_list::entry::abandon(); + } + + hazard_era* alloc_hazard_era(hint& hint, era_t era) { + if (last_hazard_era && last_era == era) { + last_hazard_era->add_guard(); + return last_hazard_era; + } + auto result = hint; + if (result == nullptr) { + result = self().need_more_hes(); + } + + hint = result->get_link(); + result->set_era(era); + result->add_guard(); + + last_hazard_era = result; + last_era = era; + return result; + } + + void release_hazard_era(hazard_era*& he, hint& hint) { + assert(he != nullptr); + if (he->release_guard() == 0) { + if (he == last_hazard_era) { + last_hazard_era = nullptr; + } + + he->set_link(hint); + hint = he; + } + he = nullptr; + } + + protected: + Derived& self() { return static_cast(*this); } + + hazard_era* begin() { return &eras[0]; } + hazard_era* end() { return &eras[Strategy::K]; } + [[nodiscard]] const hazard_era* begin() const { return &eras[0]; } + [[nodiscard]] const hazard_era* end() const { return &eras[Strategy::K]; } + + template + static hazard_era* initialize_block(T& block) { + auto begin = block.begin(); + auto end = block.end() - 1; // the last element is handled specially, so loop only over n-1 entries + for (auto it = begin; it != end;) { + auto next = it + 1; + it->set_link(next); + it = next; + } + end->set_link(block.initialize_next_block()); + return begin; + } + + static void + gather_protected_eras(std::vector& protected_eras, const hazard_era* begin, const hazard_era* end) { + for (auto it = begin; it != end; ++it) { + era_t era; + if (it->try_get_era(era)) { + protected_eras.push_back(era); + } + } + } + + hazard_era* last_hazard_era; + era_t last_era; + hazard_era eras[Strategy::K]; + }; + + template + struct static_he_thread_control_block : + basic_he_thread_control_block> { + using base = basic_he_thread_control_block; + using hazard_era = typename base::hazard_era; + using era_t = typename base::era_t; + friend base; + + void gather_protected_eras(std::vector& protected_eras) const { + base::gather_protected_eras(protected_eras, this->begin(), this->end()); + } + + private: + hazard_era* need_more_hes() { throw bad_hazard_era_alloc("hazard era pool exceeded"); } + [[nodiscard]] constexpr size_t number_of_hes() const { return Strategy::K; } + [[nodiscard]] constexpr hazard_era* initialize_next_block() const { return nullptr; } + }; + + template + struct dynamic_he_thread_control_block : + basic_he_thread_control_block> { + using base = basic_he_thread_control_block; + using hazard_era = typename base::hazard_era; + using era_t = typename base::era_t; + friend base; + + void gather_protected_eras(std::vector& protected_eras) const { + gather_protected_eras(*this, protected_eras); + } + + private: + struct alignas(64) hazard_eras_block : aligned_object { + explicit hazard_eras_block(size_t size) : size(size) { + for (auto it = begin(); it != end(); ++it) { + new (it) hazard_era; + } + } + + hazard_era* begin() { return reinterpret_cast(this + 1); } + hazard_era* end() { return begin() + size; } + + [[nodiscard]] const hazard_era* begin() const { return reinterpret_cast(this + 1); } + [[nodiscard]] const hazard_era* end() const { return begin() + size; } + + [[nodiscard]] const hazard_eras_block* next_block() const { return next; } + hazard_era* initialize_next_block() { return next ? base::initialize_block(*next) : nullptr; } + + hazard_eras_block* next = nullptr; + const size_t size; + }; + + [[nodiscard]] const hazard_eras_block* next_block() const { + // (7) - this acquire-load synchronizes-with the release-store (8) + return he_block.load(std::memory_order_acquire); + } + [[nodiscard]] size_t number_of_hes() const { return total_number_of_hes; } + hazard_era* need_more_hes() { return allocate_new_hazard_eras_block(); } + + hazard_era* initialize_next_block() { + auto block = he_block.load(std::memory_order_relaxed); + return block ? base::initialize_block(*block) : nullptr; + } + + template + static void gather_protected_eras(const T& block, std::vector& protected_eras) { + base::gather_protected_eras(protected_eras, block.begin(), block.end()); + + const auto* next = block.next_block(); + if (next) { + gather_protected_eras(*next, protected_eras); + } + } + + hazard_era* allocate_new_hazard_eras_block() { + size_t hes = std::max(static_cast(Strategy::K), total_number_of_hes / 2); + total_number_of_hes += hes; + Strategy::number_of_active_hes.fetch_add(hes, std::memory_order_relaxed); + + size_t buffer_size = sizeof(hazard_eras_block) + hes * sizeof(hazard_era); + void* buffer = hazard_eras_block::operator new(buffer_size); + auto block = ::new (buffer) hazard_eras_block(hes); + auto result = this->initialize_block(*block); + block->next = he_block.load(std::memory_order_relaxed); + // (8) - this release-store synchronizes-with the acquire-load (7) + he_block.store(block, std::memory_order_release); + return result; + } + + size_t total_number_of_hes = Strategy::K; + std::atomic he_block; + }; +} // namespace detail + +template +struct alignas(64) hazard_eras::thread_data : aligned_object { + using HE = typename thread_control_block::hazard_era*; + + ~thread_data() { + if (retire_list != nullptr) { + scan(); + if (retire_list != nullptr) { + global_thread_block_list.abandon_retired_nodes(retire_list); + } + retire_list = nullptr; + } + + if (control_block != nullptr) { + global_thread_block_list.release_entry(control_block); + control_block = nullptr; + } + } + + HE alloc_hazard_era(era_t era) { + ensure_has_control_block(); + return control_block->alloc_hazard_era(hint, era); + } + + void release_hazard_era(HE& he) { + if (he) { + assert(control_block != nullptr); + control_block->release_hazard_era(he, hint); + } + } + + std::size_t add_retired_node(detail::deletable_object_with_eras* p) { + p->next = retire_list; + retire_list = p; + return ++number_of_retired_nodes; + } + + void scan() { + std::vector protected_eras; + protected_eras.reserve(allocation_strategy::number_of_active_hazard_eras()); + + // (9) - this seq_cst-fence enforces a total order with the seq_cst-fence (5) + XENIUM_THREAD_FENCE(std::memory_order_seq_cst); + + auto adopted_nodes = global_thread_block_list.adopt_abandoned_retired_nodes(); + + std::for_each( + global_thread_block_list.begin(), global_thread_block_list.end(), [&protected_eras](const auto& entry) { + // TSan does not support explicit fences, so we cannot rely on the acquire-fence (10) + // but have to perform an acquire-load here to avoid false positives. + constexpr auto memory_order = TSAN_MEMORY_ORDER(std::memory_order_acquire, std::memory_order_relaxed); + if (entry.is_active(memory_order)) { + entry.gather_protected_eras(protected_eras); + } + }); + + // (10) - this acquire-fence synchronizes-with the release-store (4, 6) + XENIUM_THREAD_FENCE(std::memory_order_acquire); + + std::sort(protected_eras.begin(), protected_eras.end()); + auto last = std::unique(protected_eras.begin(), protected_eras.end()); + protected_eras.erase(last, protected_eras.end()); + + auto* list = retire_list; + retire_list = nullptr; + number_of_retired_nodes = 0; + reclaim_nodes(list, protected_eras); + reclaim_nodes(adopted_nodes, protected_eras); + } + +private: + void ensure_has_control_block() { + if (control_block == nullptr) { + control_block = global_thread_block_list.acquire_inactive_entry(); + control_block->initialize(hint); + control_block->activate(); + } + } + + void reclaim_nodes(detail::deletable_object_with_eras* list, const std::vector& protected_eras) { + while (list != nullptr) { + auto* cur = list; + list = list->next; + + auto era_it = std::lower_bound(protected_eras.begin(), protected_eras.end(), cur->construction_era); + if (era_it == protected_eras.end() || *era_it > cur->retirement_era) { + cur->delete_self(); + } else { + add_retired_node(cur); + } + } + } + + detail::deletable_object_with_eras* retire_list = nullptr; + std::size_t number_of_retired_nodes = 0; + typename thread_control_block::hint hint; + + thread_control_block* control_block = nullptr; + + friend class hazard_eras; + ALLOCATION_COUNTER(hazard_eras); +}; + +template +inline typename hazard_eras::thread_data& hazard_eras::local_thread_data() { + // workaround for a Clang-8 issue that causes multiple re-initializations of thread_local variables + static thread_local thread_data local_thread_data; + return local_thread_data; +} + +#ifdef TRACK_ALLOCATIONS +template +inline void hazard_eras::count_allocation() { + local_thread_data().allocation_counter.count_allocation(); +} + +template +inline void hazard_eras::count_reclamation() { + local_thread_data().allocation_counter.count_reclamation(); +} +#endif +} // namespace xenium::reclamation + +#ifdef _MSC_VER + #pragma warning(pop) +#endif diff --git a/xenium/reclamation/impl/hazard_pointer.hpp b/include/xenium/reclamation/impl/hazard_pointer.hpp similarity index 97% rename from xenium/reclamation/impl/hazard_pointer.hpp rename to include/xenium/reclamation/impl/hazard_pointer.hpp index 7a1b2d2..6c9d3c1 100644 --- a/xenium/reclamation/impl/hazard_pointer.hpp +++ b/include/xenium/reclamation/impl/hazard_pointer.hpp @@ -1,465 +1,465 @@ -// -// Copyright (c) 2018-2020 Manuel Pöter. -// Licensed under the MIT License. See LICENSE file in the project root for full license information. -// - -#ifndef HAZARD_POINTER_IMPL - #error "This is an impl file and must not be included directly!" -#endif - -#include -#include - -#include -#include -#include - -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4324) // structure was padded due to alignment specifier -#endif - -namespace xenium::reclamation { - -template -template -hazard_pointer::guard_ptr::guard_ptr(const MarkedPtr& p) : base(p), hp() { - if (this->ptr.get() != nullptr) { - hp = local_thread_data.alloc_hazard_pointer(); - hp->set_object(this->ptr.get()); - } -} - -template -template -hazard_pointer::guard_ptr::guard_ptr(const guard_ptr& p) : guard_ptr(p.ptr) {} - -template -template -hazard_pointer::guard_ptr::guard_ptr(guard_ptr&& p) noexcept : base(p.ptr), hp(p.hp) { - p.ptr.reset(); - p.hp = nullptr; -} - -template -template -auto hazard_pointer::guard_ptr::operator=(const guard_ptr& p) -> guard_ptr& { - if (&p == this) { - return *this; - } - - if (hp == nullptr) { - hp = local_thread_data.alloc_hazard_pointer(); - } - this->ptr = p.ptr; - hp->set_object(this->ptr.get()); - return *this; -} - -template -template -auto hazard_pointer::guard_ptr::operator=(guard_ptr&& p) noexcept -> guard_ptr& { - if (&p == this) { - return *this; - } - - reset(); - this->ptr = std::move(p.ptr); - hp = p.hp; - p.ptr.reset(); - p.hp = nullptr; - return *this; -} - -template -template -void hazard_pointer::guard_ptr::acquire(const concurrent_ptr& p, std::memory_order order) { - auto p1 = p.load(std::memory_order_relaxed); - if (p1 == this->ptr) { - return; - } - if (p1 != nullptr && hp == nullptr) { - hp = local_thread_data.alloc_hazard_pointer(); - } - auto p2 = p1; - do { - if (p2 == nullptr) { - reset(); - return; - } - - p1 = p2; - hp->set_object(p1.get()); - // (1) - this load operation potentially synchronizes-with any release operation on p. - p2 = p.load(order); - } while (p1.get() != p2.get()); - - this->ptr = p2; -} - -template -template -bool hazard_pointer::guard_ptr::acquire_if_equal(const concurrent_ptr& p, - const MarkedPtr& expected, - std::memory_order order) { - auto p1 = p.load(std::memory_order_relaxed); - if (p1 == nullptr || p1 != expected) { - reset(); - return p1 == expected; - } - - if (hp == nullptr) { - hp = local_thread_data.alloc_hazard_pointer(); - } - hp->set_object(p1.get()); - // (2) - this load operation potentially synchronizes-with any release operation on p. - this->ptr = p.load(order); - if (this->ptr != p1) { - reset(); - return false; - } - return true; -} - -template -template -void hazard_pointer::guard_ptr::reset() noexcept { - local_thread_data.release_hazard_pointer(hp); - this->ptr.reset(); -} - -template -template -void hazard_pointer::guard_ptr::do_swap(guard_ptr& g) noexcept { - std::swap(hp, g.hp); -} - -template -template -void hazard_pointer::guard_ptr::reclaim(Deleter d) noexcept { - auto* p = this->ptr.get(); - reset(); - p->set_deleter(std::move(d)); - if (local_thread_data.add_retired_node(p) >= allocation_strategy::retired_nodes_threshold()) { - local_thread_data.scan(); - } -} - -namespace detail { - template - struct alignas(64) basic_hp_thread_control_block : - detail::thread_block_list::entry, - aligned_object> { - struct hazard_pointer { - void set_object(detail::deletable_object* obj) { - // (3) - this release-store synchronizes-with the acquire-fence (9) - value.store(reinterpret_cast(obj), std::memory_order_release); - // This release is required because when acquire/acquire_if_equal is called on a - // guard_ptr with with an active HE entry, set_era is called without an intermediate - // call to set_link, i.e., the protected era is updated. This ensures the required - // happens-before relation between releasing a guard_ptr to a node and reclamation - // of that node. - - // (4) - this seq_cst-fence enforces a total order with the seq_cst-fence (8) - XENIUM_THREAD_FENCE(std::memory_order_seq_cst); - } - - bool try_get_object(detail::deletable_object*& result) const { - // TSan does not support explicit fences, so we cannot rely on the acquire-fence (9) - // but have to perform an acquire-load here to avoid false positives. - constexpr auto memory_order = TSAN_MEMORY_ORDER(std::memory_order_acquire, std::memory_order_relaxed); - auto v = value.load(memory_order); - if (v.mark() == 0) { - result = reinterpret_cast(v.get()); - return true; - } - return false; // value contains a link - } - - void set_link(hazard_pointer* link) { - // (5) - this release store synchronizes-with the acquire fence (9) - value.store(marked_ptr(reinterpret_cast(link), 1), std::memory_order_release); - } - [[nodiscard]] hazard_pointer* get_link() const { - assert(is_link()); - return reinterpret_cast(value.load(std::memory_order_relaxed).get()); - } - - [[nodiscard]] bool is_link() const { return value.load(std::memory_order_relaxed).mark() != 0; } - - private: - // since we use the hazard pointer array to build our internal linked list of hazard pointers - // we set the LSB to signal that this is an internal pointer and not a pointer to a protected object. - std::atomic> value; - }; - - using hint = hazard_pointer*; - - void initialize(hint& hint) { - Strategy::number_of_active_hps.fetch_add(self().number_of_hps(), std::memory_order_relaxed); - hint = initialize_block(self()); - } - - void abandon() { - Strategy::number_of_active_hps.fetch_sub(self().number_of_hps(), std::memory_order_relaxed); - detail::thread_block_list::entry::abandon(); - } - - hazard_pointer* alloc_hazard_pointer(hint& hint) { - auto result = hint; - if (result == nullptr) { - result = self().need_more_hps(); - } - - hint = result->get_link(); - return result; - } - - void release_hazard_pointer(hazard_pointer*& hp, hint& hint) { - if (hp != nullptr) { - hp->set_link(hint); - hint = hp; - hp = nullptr; - } - } - - protected: - Derived& self() { return static_cast(*this); } - - hazard_pointer* begin() { return &pointers[0]; } - hazard_pointer* end() { return &pointers[Strategy::K]; } - [[nodiscard]] const hazard_pointer* begin() const { return &pointers[0]; } - [[nodiscard]] const hazard_pointer* end() const { return &pointers[Strategy::K]; } - - template - static hazard_pointer* initialize_block(T& block) { - auto begin = block.begin(); - auto end = block.end() - 1; // the last element is handled specially, so loop only over n-1 entries - for (auto it = begin; it != end;) { - auto next = it + 1; - it->set_link(next); - it = next; - } - end->set_link(block.initialize_next_block()); - return begin; - } - - static void gather_protected_pointers(std::vector& protected_ptrs, - const hazard_pointer* begin, - const hazard_pointer* end) { - for (auto it = begin; it != end; ++it) { - detail::deletable_object* obj; - if (it->try_get_object(obj)) { - protected_ptrs.push_back(obj); - } - } - } - - hazard_pointer pointers[Strategy::K]; - }; - - template - struct static_hp_thread_control_block : - basic_hp_thread_control_block> { - using base = basic_hp_thread_control_block; - using hazard_pointer = typename base::hazard_pointer; - friend base; - - void gather_protected_pointers(std::vector& protected_ptrs) const { - base::gather_protected_pointers(protected_ptrs, this->begin(), this->end()); - } - - private: - hazard_pointer* need_more_hps() { throw bad_hazard_pointer_alloc("hazard pointer pool exceeded"); } - [[nodiscard]] constexpr size_t number_of_hps() const { return Strategy::K; } - [[nodiscard]] constexpr hazard_pointer* initialize_next_block() const { return nullptr; } - }; - - template - struct dynamic_hp_thread_control_block : - basic_hp_thread_control_block> { - using base = basic_hp_thread_control_block; - using hazard_pointer = typename base::hazard_pointer; - friend base; - - void gather_protected_pointers(std::vector& protected_ptrs) const { - gather_protected_pointers(*this, protected_ptrs); - } - - private: - struct alignas(64) hazard_pointer_block : aligned_object { - explicit hazard_pointer_block(size_t size) : size(size) {} - - hazard_pointer* begin() { return reinterpret_cast(this + 1); } - hazard_pointer* end() { return begin() + size; } - - [[nodiscard]] const hazard_pointer* begin() const { return reinterpret_cast(this + 1); } - [[nodiscard]] const hazard_pointer* end() const { return begin() + size; } - - [[nodiscard]] const hazard_pointer_block* next_block() const { return next; } - hazard_pointer* initialize_next_block() { return next ? base::initialize_block(*next) : nullptr; } - - hazard_pointer_block* next = nullptr; - const size_t size; - }; - - [[nodiscard]] const hazard_pointer_block* next_block() const { - // (6) - this acquire-load synchronizes-with the release-store (7) - return hp_block.load(std::memory_order_acquire); - } - [[nodiscard]] size_t number_of_hps() const { return total_number_of_hps; } - hazard_pointer* need_more_hps() { return allocate_new_hazard_pointer_block(); } - - hazard_pointer* initialize_next_block() { - auto block = hp_block.load(std::memory_order_relaxed); - return block ? base::initialize_block(*block) : nullptr; - } - - template - static void gather_protected_pointers(const T& block, - std::vector& protected_ptrs) { - base::gather_protected_pointers(protected_ptrs, block.begin(), block.end()); - - const auto* next = block.next_block(); - if (next) { - gather_protected_pointers(*next, protected_ptrs); - } - } - - static detail::deletable_object* as_internal_pointer(hazard_pointer* p) { - // since we use the hazard pointer array to build our internal linked list of hazard pointers - // we set the LSB to signal that this is an internal pointer and not a pointer to a protected object. - auto marked = reinterpret_cast(p) | 1; - return reinterpret_cast(marked); - } - - hazard_pointer* allocate_new_hazard_pointer_block() { - size_t hps = std::max(static_cast(Strategy::K), total_number_of_hps / 2); - total_number_of_hps += hps; - Strategy::number_of_active_hps.fetch_add(hps, std::memory_order_relaxed); - - size_t buffer_size = sizeof(hazard_pointer_block) + hps * sizeof(hazard_pointer); - void* buffer = hazard_pointer_block::operator new(buffer_size); - auto block = ::new (buffer) hazard_pointer_block(hps); - auto result = this->initialize_block(*block); - block->next = hp_block.load(std::memory_order_relaxed); - // (7) - this release-store synchronizes-with the acquire-load (6) - hp_block.store(block, std::memory_order_release); - return result; - } - - size_t total_number_of_hps = Strategy::K; - std::atomic hp_block; - }; -} // namespace detail - -template -struct alignas(64) hazard_pointer::thread_data : aligned_object { - using HP = typename thread_control_block::hazard_pointer*; - - ~thread_data() { - if (retire_list != nullptr) { - scan(); - if (retire_list != nullptr) { - global_thread_block_list.abandon_retired_nodes(retire_list); - } - retire_list = nullptr; - } - - if (control_block != nullptr) { - global_thread_block_list.release_entry(control_block); - control_block = nullptr; - } - } - - HP alloc_hazard_pointer() { - ensure_has_control_block(); - return control_block->alloc_hazard_pointer(hint); - } - - void release_hazard_pointer(HP& hp) { control_block->release_hazard_pointer(hp, hint); } - - std::size_t add_retired_node(detail::deletable_object* p) { - p->next = retire_list; - retire_list = p; - return ++number_of_retired_nodes; - } - - void scan() { - std::vector protected_pointers; - protected_pointers.reserve(allocation_strategy::number_of_active_hazard_pointers()); - - // (8) - this seq_cst-fence enforces a total order with the seq_cst-fence (4) - XENIUM_THREAD_FENCE(std::memory_order_seq_cst); - - auto adopted_nodes = global_thread_block_list.adopt_abandoned_retired_nodes(); - - std::for_each( - global_thread_block_list.begin(), global_thread_block_list.end(), [&protected_pointers](const auto& entry) { - // TSan does not support explicit fences, so we cannot rely on the acquire-fence (9) - // but have to perform an acquire-load here to avoid false positives. - constexpr auto memory_order = TSAN_MEMORY_ORDER(std::memory_order_acquire, std::memory_order_relaxed); - if (entry.is_active(memory_order)) { - entry.gather_protected_pointers(protected_pointers); - } - }); - - // (9) - this acquire-fence synchronizes-with the release-store (3, 5) - XENIUM_THREAD_FENCE(std::memory_order_acquire); - - std::sort(protected_pointers.begin(), protected_pointers.end()); - - auto* list = retire_list; - retire_list = nullptr; - number_of_retired_nodes = 0; - reclaim_nodes(list, protected_pointers); - reclaim_nodes(adopted_nodes, protected_pointers); - } - -private: - void ensure_has_control_block() { - if (control_block == nullptr) { - control_block = global_thread_block_list.acquire_entry(); - control_block->initialize(hint); - } - } - - void reclaim_nodes(detail::deletable_object* list, - const std::vector& protected_pointers) { - while (list != nullptr) { - auto* cur = list; - list = list->next; - - if (std::binary_search(protected_pointers.begin(), protected_pointers.end(), cur)) { - add_retired_node(cur); - } else { - cur->delete_self(); - } - } - } - detail::deletable_object* retire_list = nullptr; - std::size_t number_of_retired_nodes = 0; - typename thread_control_block::hint hint{}; - - thread_control_block* control_block = nullptr; - - friend class hazard_pointer; - ALLOCATION_COUNTER(hazard_pointer); -}; - -#ifdef TRACK_ALLOCATIONS -template -inline void hazard_pointer::count_allocation() { - local_thread_data.allocation_counter.count_allocation(); -} - -template -inline void hazard_pointer::count_reclamation() { - local_thread_data.allocation_counter.count_reclamation(); -} -#endif -} // namespace xenium::reclamation - -#ifdef _MSC_VER - #pragma warning(pop) -#endif +// +// Copyright (c) 2018-2020 Manuel Pöter. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +#ifndef HAZARD_POINTER_IMPL + #error "This is an impl file and must not be included directly!" +#endif + +#include +#include + +#include +#include +#include + +#ifdef _MSC_VER + #pragma warning(push) + #pragma warning(disable : 4324) // structure was padded due to alignment specifier +#endif + +namespace xenium::reclamation { + +template +template +hazard_pointer::guard_ptr::guard_ptr(const MarkedPtr& p) : base(p), hp() { + if (this->ptr.get() != nullptr) { + hp = local_thread_data.alloc_hazard_pointer(); + hp->set_object(this->ptr.get()); + } +} + +template +template +hazard_pointer::guard_ptr::guard_ptr(const guard_ptr& p) : guard_ptr(p.ptr) {} + +template +template +hazard_pointer::guard_ptr::guard_ptr(guard_ptr&& p) noexcept : base(p.ptr), hp(p.hp) { + p.ptr.reset(); + p.hp = nullptr; +} + +template +template +auto hazard_pointer::guard_ptr::operator=(const guard_ptr& p) -> guard_ptr& { + if (&p == this) { + return *this; + } + + if (hp == nullptr) { + hp = local_thread_data.alloc_hazard_pointer(); + } + this->ptr = p.ptr; + hp->set_object(this->ptr.get()); + return *this; +} + +template +template +auto hazard_pointer::guard_ptr::operator=(guard_ptr&& p) noexcept -> guard_ptr& { + if (&p == this) { + return *this; + } + + reset(); + this->ptr = std::move(p.ptr); + hp = p.hp; + p.ptr.reset(); + p.hp = nullptr; + return *this; +} + +template +template +void hazard_pointer::guard_ptr::acquire(const concurrent_ptr& p, std::memory_order order) { + auto p1 = p.load(std::memory_order_relaxed); + if (p1 == this->ptr) { + return; + } + if (p1 != nullptr && hp == nullptr) { + hp = local_thread_data.alloc_hazard_pointer(); + } + auto p2 = p1; + do { + if (p2 == nullptr) { + reset(); + return; + } + + p1 = p2; + hp->set_object(p1.get()); + // (1) - this load operation potentially synchronizes-with any release operation on p. + p2 = p.load(order); + } while (p1.get() != p2.get()); + + this->ptr = p2; +} + +template +template +bool hazard_pointer::guard_ptr::acquire_if_equal(const concurrent_ptr& p, + const MarkedPtr& expected, + std::memory_order order) { + auto p1 = p.load(std::memory_order_relaxed); + if (p1 == nullptr || p1 != expected) { + reset(); + return p1 == expected; + } + + if (hp == nullptr) { + hp = local_thread_data.alloc_hazard_pointer(); + } + hp->set_object(p1.get()); + // (2) - this load operation potentially synchronizes-with any release operation on p. + this->ptr = p.load(order); + if (this->ptr != p1) { + reset(); + return false; + } + return true; +} + +template +template +void hazard_pointer::guard_ptr::reset() noexcept { + local_thread_data.release_hazard_pointer(hp); + this->ptr.reset(); +} + +template +template +void hazard_pointer::guard_ptr::do_swap(guard_ptr& g) noexcept { + std::swap(hp, g.hp); +} + +template +template +void hazard_pointer::guard_ptr::reclaim(Deleter d) noexcept { + auto* p = this->ptr.get(); + reset(); + p->set_deleter(std::move(d)); + if (local_thread_data.add_retired_node(p) >= allocation_strategy::retired_nodes_threshold()) { + local_thread_data.scan(); + } +} + +namespace detail { + template + struct alignas(64) basic_hp_thread_control_block : + detail::thread_block_list::entry, + aligned_object> { + struct hazard_pointer { + void set_object(detail::deletable_object* obj) { + // (3) - this release-store synchronizes-with the acquire-fence (9) + value.store(reinterpret_cast(obj), std::memory_order_release); + // This release is required because when acquire/acquire_if_equal is called on a + // guard_ptr with with an active HE entry, set_era is called without an intermediate + // call to set_link, i.e., the protected era is updated. This ensures the required + // happens-before relation between releasing a guard_ptr to a node and reclamation + // of that node. + + // (4) - this seq_cst-fence enforces a total order with the seq_cst-fence (8) + XENIUM_THREAD_FENCE(std::memory_order_seq_cst); + } + + bool try_get_object(detail::deletable_object*& result) const { + // TSan does not support explicit fences, so we cannot rely on the acquire-fence (9) + // but have to perform an acquire-load here to avoid false positives. + constexpr auto memory_order = TSAN_MEMORY_ORDER(std::memory_order_acquire, std::memory_order_relaxed); + auto v = value.load(memory_order); + if (v.mark() == 0) { + result = reinterpret_cast(v.get()); + return true; + } + return false; // value contains a link + } + + void set_link(hazard_pointer* link) { + // (5) - this release store synchronizes-with the acquire fence (9) + value.store(marked_ptr(reinterpret_cast(link), 1), std::memory_order_release); + } + [[nodiscard]] hazard_pointer* get_link() const { + assert(is_link()); + return reinterpret_cast(value.load(std::memory_order_relaxed).get()); + } + + [[nodiscard]] bool is_link() const { return value.load(std::memory_order_relaxed).mark() != 0; } + + private: + // since we use the hazard pointer array to build our internal linked list of hazard pointers + // we set the LSB to signal that this is an internal pointer and not a pointer to a protected object. + std::atomic> value; + }; + + using hint = hazard_pointer*; + + void initialize(hint& hint) { + Strategy::number_of_active_hps.fetch_add(self().number_of_hps(), std::memory_order_relaxed); + hint = initialize_block(self()); + } + + void abandon() { + Strategy::number_of_active_hps.fetch_sub(self().number_of_hps(), std::memory_order_relaxed); + detail::thread_block_list::entry::abandon(); + } + + hazard_pointer* alloc_hazard_pointer(hint& hint) { + auto result = hint; + if (result == nullptr) { + result = self().need_more_hps(); + } + + hint = result->get_link(); + return result; + } + + void release_hazard_pointer(hazard_pointer*& hp, hint& hint) { + if (hp != nullptr) { + hp->set_link(hint); + hint = hp; + hp = nullptr; + } + } + + protected: + Derived& self() { return static_cast(*this); } + + hazard_pointer* begin() { return &pointers[0]; } + hazard_pointer* end() { return &pointers[Strategy::K]; } + [[nodiscard]] const hazard_pointer* begin() const { return &pointers[0]; } + [[nodiscard]] const hazard_pointer* end() const { return &pointers[Strategy::K]; } + + template + static hazard_pointer* initialize_block(T& block) { + auto begin = block.begin(); + auto end = block.end() - 1; // the last element is handled specially, so loop only over n-1 entries + for (auto it = begin; it != end;) { + auto next = it + 1; + it->set_link(next); + it = next; + } + end->set_link(block.initialize_next_block()); + return begin; + } + + static void gather_protected_pointers(std::vector& protected_ptrs, + const hazard_pointer* begin, + const hazard_pointer* end) { + for (auto it = begin; it != end; ++it) { + detail::deletable_object* obj; + if (it->try_get_object(obj)) { + protected_ptrs.push_back(obj); + } + } + } + + hazard_pointer pointers[Strategy::K]; + }; + + template + struct static_hp_thread_control_block : + basic_hp_thread_control_block> { + using base = basic_hp_thread_control_block; + using hazard_pointer = typename base::hazard_pointer; + friend base; + + void gather_protected_pointers(std::vector& protected_ptrs) const { + base::gather_protected_pointers(protected_ptrs, this->begin(), this->end()); + } + + private: + hazard_pointer* need_more_hps() { throw bad_hazard_pointer_alloc("hazard pointer pool exceeded"); } + [[nodiscard]] constexpr size_t number_of_hps() const { return Strategy::K; } + [[nodiscard]] constexpr hazard_pointer* initialize_next_block() const { return nullptr; } + }; + + template + struct dynamic_hp_thread_control_block : + basic_hp_thread_control_block> { + using base = basic_hp_thread_control_block; + using hazard_pointer = typename base::hazard_pointer; + friend base; + + void gather_protected_pointers(std::vector& protected_ptrs) const { + gather_protected_pointers(*this, protected_ptrs); + } + + private: + struct alignas(64) hazard_pointer_block : aligned_object { + explicit hazard_pointer_block(size_t size) : size(size) {} + + hazard_pointer* begin() { return reinterpret_cast(this + 1); } + hazard_pointer* end() { return begin() + size; } + + [[nodiscard]] const hazard_pointer* begin() const { return reinterpret_cast(this + 1); } + [[nodiscard]] const hazard_pointer* end() const { return begin() + size; } + + [[nodiscard]] const hazard_pointer_block* next_block() const { return next; } + hazard_pointer* initialize_next_block() { return next ? base::initialize_block(*next) : nullptr; } + + hazard_pointer_block* next = nullptr; + const size_t size; + }; + + [[nodiscard]] const hazard_pointer_block* next_block() const { + // (6) - this acquire-load synchronizes-with the release-store (7) + return hp_block.load(std::memory_order_acquire); + } + [[nodiscard]] size_t number_of_hps() const { return total_number_of_hps; } + hazard_pointer* need_more_hps() { return allocate_new_hazard_pointer_block(); } + + hazard_pointer* initialize_next_block() { + auto block = hp_block.load(std::memory_order_relaxed); + return block ? base::initialize_block(*block) : nullptr; + } + + template + static void gather_protected_pointers(const T& block, + std::vector& protected_ptrs) { + base::gather_protected_pointers(protected_ptrs, block.begin(), block.end()); + + const auto* next = block.next_block(); + if (next) { + gather_protected_pointers(*next, protected_ptrs); + } + } + + static detail::deletable_object* as_internal_pointer(hazard_pointer* p) { + // since we use the hazard pointer array to build our internal linked list of hazard pointers + // we set the LSB to signal that this is an internal pointer and not a pointer to a protected object. + auto marked = reinterpret_cast(p) | 1; + return reinterpret_cast(marked); + } + + hazard_pointer* allocate_new_hazard_pointer_block() { + size_t hps = std::max(static_cast(Strategy::K), total_number_of_hps / 2); + total_number_of_hps += hps; + Strategy::number_of_active_hps.fetch_add(hps, std::memory_order_relaxed); + + size_t buffer_size = sizeof(hazard_pointer_block) + hps * sizeof(hazard_pointer); + void* buffer = hazard_pointer_block::operator new(buffer_size); + auto block = ::new (buffer) hazard_pointer_block(hps); + auto result = this->initialize_block(*block); + block->next = hp_block.load(std::memory_order_relaxed); + // (7) - this release-store synchronizes-with the acquire-load (6) + hp_block.store(block, std::memory_order_release); + return result; + } + + size_t total_number_of_hps = Strategy::K; + std::atomic hp_block; + }; +} // namespace detail + +template +struct alignas(64) hazard_pointer::thread_data : aligned_object { + using HP = typename thread_control_block::hazard_pointer*; + + ~thread_data() { + if (retire_list != nullptr) { + scan(); + if (retire_list != nullptr) { + global_thread_block_list.abandon_retired_nodes(retire_list); + } + retire_list = nullptr; + } + + if (control_block != nullptr) { + global_thread_block_list.release_entry(control_block); + control_block = nullptr; + } + } + + HP alloc_hazard_pointer() { + ensure_has_control_block(); + return control_block->alloc_hazard_pointer(hint); + } + + void release_hazard_pointer(HP& hp) { control_block->release_hazard_pointer(hp, hint); } + + std::size_t add_retired_node(detail::deletable_object* p) { + p->next = retire_list; + retire_list = p; + return ++number_of_retired_nodes; + } + + void scan() { + std::vector protected_pointers; + protected_pointers.reserve(allocation_strategy::number_of_active_hazard_pointers()); + + // (8) - this seq_cst-fence enforces a total order with the seq_cst-fence (4) + XENIUM_THREAD_FENCE(std::memory_order_seq_cst); + + auto adopted_nodes = global_thread_block_list.adopt_abandoned_retired_nodes(); + + std::for_each( + global_thread_block_list.begin(), global_thread_block_list.end(), [&protected_pointers](const auto& entry) { + // TSan does not support explicit fences, so we cannot rely on the acquire-fence (9) + // but have to perform an acquire-load here to avoid false positives. + constexpr auto memory_order = TSAN_MEMORY_ORDER(std::memory_order_acquire, std::memory_order_relaxed); + if (entry.is_active(memory_order)) { + entry.gather_protected_pointers(protected_pointers); + } + }); + + // (9) - this acquire-fence synchronizes-with the release-store (3, 5) + XENIUM_THREAD_FENCE(std::memory_order_acquire); + + std::sort(protected_pointers.begin(), protected_pointers.end()); + + auto* list = retire_list; + retire_list = nullptr; + number_of_retired_nodes = 0; + reclaim_nodes(list, protected_pointers); + reclaim_nodes(adopted_nodes, protected_pointers); + } + +private: + void ensure_has_control_block() { + if (control_block == nullptr) { + control_block = global_thread_block_list.acquire_entry(); + control_block->initialize(hint); + } + } + + void reclaim_nodes(detail::deletable_object* list, + const std::vector& protected_pointers) { + while (list != nullptr) { + auto* cur = list; + list = list->next; + + if (std::binary_search(protected_pointers.begin(), protected_pointers.end(), cur)) { + add_retired_node(cur); + } else { + cur->delete_self(); + } + } + } + detail::deletable_object* retire_list = nullptr; + std::size_t number_of_retired_nodes = 0; + typename thread_control_block::hint hint{}; + + thread_control_block* control_block = nullptr; + + friend class hazard_pointer; + ALLOCATION_COUNTER(hazard_pointer); +}; + +#ifdef TRACK_ALLOCATIONS +template +inline void hazard_pointer::count_allocation() { + local_thread_data.allocation_counter.count_allocation(); +} + +template +inline void hazard_pointer::count_reclamation() { + local_thread_data.allocation_counter.count_reclamation(); +} +#endif +} // namespace xenium::reclamation + +#ifdef _MSC_VER + #pragma warning(pop) +#endif diff --git a/xenium/reclamation/impl/lock_free_ref_count.hpp b/include/xenium/reclamation/impl/lock_free_ref_count.hpp similarity index 97% rename from xenium/reclamation/impl/lock_free_ref_count.hpp rename to include/xenium/reclamation/impl/lock_free_ref_count.hpp index 46a8fc1..a716ffa 100644 --- a/xenium/reclamation/impl/lock_free_ref_count.hpp +++ b/include/xenium/reclamation/impl/lock_free_ref_count.hpp @@ -1,342 +1,342 @@ -// -// Copyright (c) 2018-2020 Manuel Pöter. -// Licensed under the MIT License. See LICENSE file in the project root for full license information. -// - -#ifndef LOCK_FREE_REF_COUNT_IMPL - #error "This is an impl file and must not be included directly!" -#endif - -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4127) // conditional expression is constant -#endif - -namespace xenium::reclamation { - -template -template -class lock_free_ref_count::enable_concurrent_ptr::free_list { -public: - T* pop() { - if (max_local_elements > 0) { - if (auto* result = local_free_list().pop()) { - return result; - } - } - - guard_ptr guard; - - while (true) { - // (1) - this acquire-load synchronizes-with the release-CAS (3) - guard = acquire_guard(head, std::memory_order_acquire); - if (guard.get() == nullptr) { - return nullptr; - } - - // Note: ref_count can be anything here since multiple threads - // could have gotten a reference to the node on the freelist. - marked_ptr expected(guard); - auto next = guard->next_free().load(std::memory_order_relaxed); - // since head is only changed via CAS operations it is sufficient to use relaxed order - // for this operation as it is always part of a release-sequence headed by (3) - if (head.compare_exchange_weak(expected, next, std::memory_order_relaxed)) { - assert((guard->ref_count().load(std::memory_order_relaxed) & RefCountClaimBit) != 0 && - "ClaimBit must be set for a node on the free list"); - - auto* ptr = guard.get(); - ptr->ref_count().fetch_sub(RefCountClaimBit, std::memory_order_relaxed); // clear claim bit - ptr->next_free().store(nullptr, std::memory_order_relaxed); - guard.ptr.reset(); // reset guard_ptr to prevent decrement of ref_count - return ptr; - } - } - } - - void push(T* node) { - assert(node->ref_count().load(std::memory_order_relaxed) & RefCountClaimBit && - "ClaimBit must be set for a node to be put on the free list"); - if (max_local_elements > 0 && local_free_list().push(node)) { - return; - } - - add_nodes(node, node); - } - -private: - void add_nodes(T* first, T* last) { - // (2) - this acquire-load synchronizes-with the release-CAS (3) - auto old = head.load(std::memory_order_acquire); - do { - last->next_free().store(old, std::memory_order_relaxed); - // (3) - if this release-CAS succeeds, it synchronizes-with the acquire-loads (1, 2) - // if it failes, the reload synchronizes-with itself (3) - } while (!head.compare_exchange_weak(old, first, std::memory_order_release, std::memory_order_acquire)); - } - - // the free list is implemented as a FILO single linked list - // the LSB of a node's ref_count acts as claim bit, so for all nodes on the free list the bit has to be set - concurrent_ptr head; - - class thread_local_free_list { - public: - ~thread_local_free_list() noexcept { - if (head == nullptr) { - return; - } - auto* first = head; - auto* last = head; - auto next = last->next_free().load(std::memory_order_relaxed); - while (next) { - last = next.get(); - next = next->next_free().load(std::memory_order_relaxed); - } - global_free_list.add_nodes(first, last); - } - - bool push(T* node) { - if (number_of_elements >= max_local_elements) { - return false; - } - node->next_free().store(head, std::memory_order_relaxed); - head = node; - ++number_of_elements; - return true; - } - - T* pop() { - auto* result = head; - if (result) { - assert(number_of_elements > 0); - head = result->next_free().load(std::memory_order_relaxed).get(); - --number_of_elements; - // clear claim bit and increment ref_count - result->ref_count().fetch_add(RefCountInc - RefCountClaimBit, std::memory_order_relaxed); - result->next_free().store(nullptr, std::memory_order_relaxed); - } - return result; - } - - private: - size_t number_of_elements = 0; - T* head = nullptr; - }; - - static constexpr size_t max_local_elements = Traits::thread_local_free_list_size; - - static thread_local_free_list& local_free_list() { - // workaround for gcc issue causing redefinition of __tls_guard when - // defining this as static thread_local member of free_list. - alignas(64) static thread_local thread_local_free_list local_free_list; - return local_free_list; - } -}; - -template -template -void* lock_free_ref_count::enable_concurrent_ptr::operator new(size_t sz) { - assert(sz == sizeof(T) && "Cannot handle allocations of anything other than T instances"); - T* result = global_free_list.pop(); - if (result == nullptr) { - auto h = static_cast(::operator new(sz + sizeof(header))); - h->ref_count.store(RefCountInc, std::memory_order_release); - result = static_cast(static_cast(h + 1)); - } - - return result; -} - -template -template -void lock_free_ref_count::enable_concurrent_ptr::operator delete(void* p) { - auto* node = static_cast(p); - assert((node->ref_count().load(std::memory_order_relaxed) & RefCountClaimBit) == 0); - - if (node->decrement_refcnt()) { - node->push_to_free_list(); - } -} - -template -template -bool lock_free_ref_count::enable_concurrent_ptr::decrement_refcnt() { - unsigned old_refcnt, new_refcnt; - do { - old_refcnt = ref_count().load(std::memory_order_relaxed); - new_refcnt = old_refcnt - RefCountInc; - if (new_refcnt == 0) { - new_refcnt = RefCountClaimBit; - } - // (4) - this release/acquire CAS synchronizes with itself - } while (!ref_count().compare_exchange_weak(old_refcnt, - new_refcnt, - new_refcnt == RefCountClaimBit ? std::memory_order_acquire - : std::memory_order_release, - std::memory_order_relaxed)); - - // free node iff ref_count is zero AND we're the first thread to "claim" this node for reclamation. - return ((old_refcnt - new_refcnt) & RefCountClaimBit) != 0; -} - -template -template -lock_free_ref_count::guard_ptr::guard_ptr(const MarkedPtr& p) noexcept : base(p) { - if (this->ptr.get() != nullptr) { - this->ptr->ref_count().fetch_add(RefCountInc, std::memory_order_relaxed); - } -} - -template -template -lock_free_ref_count::guard_ptr::guard_ptr(const guard_ptr& p) noexcept : guard_ptr(p.ptr) {} - -template -template -lock_free_ref_count::guard_ptr::guard_ptr(guard_ptr&& p) noexcept : base(p.ptr) { - p.ptr.reset(); -} - -template -template -auto lock_free_ref_count::guard_ptr::operator=(const guard_ptr& p) -> guard_ptr& { - if (&p == this) { - return *this; - } - - reset(); - this->ptr = p.ptr; - if (this->ptr.get() != nullptr) { - this->ptr->ref_count().fetch_add(RefCountInc, std::memory_order_relaxed); - } - return *this; -} - -template -template -auto lock_free_ref_count::guard_ptr::operator=(guard_ptr&& p) noexcept -> guard_ptr& { - if (&p == this) { - return *this; - } - - reset(); - this->ptr = std::move(p.ptr); - p.ptr.reset(); - return *this; -} - -template -template -void lock_free_ref_count::guard_ptr::acquire(const concurrent_ptr& p, - std::memory_order order) noexcept { - for (;;) { - reset(); - // FIXME: If this load is relaxed, TSan reports a data race between the following - // fetch-add and the initialization of the ref_count field. I tend to disagree, as - // the fetch_add should be ordered after the initial store (operator new) in the - // modification order of ref_count. Therefore the acquire-fetch-add should - // synchronize-with the release store. - // I created a GitHub issue: - // But for now, let's make this an acquire-load to make TSan happy. - auto q = p.load(std::memory_order_acquire); - this->ptr = q; - if (q.get() == nullptr) { - return; - } - - // (5) - this acquire-fetch_add synchronizes-with the release-fetch_sub (7) - // this ensures that a change to p becomes visible - q->ref_count().fetch_add(RefCountInc, std::memory_order_acquire); - - if (q == p.load(order)) { - return; - } - } -} - -template -template -bool lock_free_ref_count::guard_ptr::acquire_if_equal(const concurrent_ptr& p, - const MarkedPtr& expected, - std::memory_order order) noexcept { - reset(); - // FIXME: same issue with TSan as in acquire (see above). - auto q = p.load(std::memory_order_acquire); - if (q != expected) { - return false; - } - - this->ptr = q; - if (q.get() == nullptr) { - return true; - } - - // (6) - this acquire-fetch_add synchronizes-with the release-fetch_sub (7) - // this ensures that a change to p becomes visible - q->ref_count().fetch_add(RefCountInc, std::memory_order_acquire); - - if (q == p.load(order)) { - return true; - } - - reset(); - return false; -} - -template -template -void lock_free_ref_count::guard_ptr::reset() noexcept { - auto* p = this->ptr.get(); - this->ptr.reset(); - if (p == nullptr) { - return; - } - - if (p->decrement_refcnt()) { - if (!p->is_destroyed()) { - p->~T(); - } - - p->push_to_free_list(); - } -} - -template -template -void lock_free_ref_count::guard_ptr::reclaim(Deleter) noexcept { - if (this->ptr.get() != nullptr) { - assert(this->ptr->refs() > 1); - // ref_count was initialized with "1", so we need an additional - // decrement to ensure that the node gets reclaimed. - // ref_count cannot drop to zero here -> no check required. - // (7) - this release-fetch-sub synchronizes-with the acquire-fetch-add (5, 6) - this->ptr->ref_count().fetch_sub(RefCountInc, std::memory_order_release); - } - reset(); -} - -template -template -typename lock_free_ref_count::template enable_concurrent_ptr::free_list - lock_free_ref_count::enable_concurrent_ptr::global_free_list; - -#ifdef TRACK_ALLOCATIONS -template -inline detail::allocation_counter& lock_free_ref_count::allocation_counter() { - return allocation_counter_; -}; - -template -inline void lock_free_ref_count::count_allocation() { - allocation_counter().count_allocation(); -} - -template -inline void lock_free_ref_count::count_reclamation() { - allocation_counter().count_reclamation(); -} -#endif -} // namespace xenium::reclamation - -#ifdef _MSC_VER - #pragma warning(pop) -#endif +// +// Copyright (c) 2018-2020 Manuel Pöter. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +#ifndef LOCK_FREE_REF_COUNT_IMPL + #error "This is an impl file and must not be included directly!" +#endif + +#ifdef _MSC_VER + #pragma warning(push) + #pragma warning(disable : 4127) // conditional expression is constant +#endif + +namespace xenium::reclamation { + +template +template +class lock_free_ref_count::enable_concurrent_ptr::free_list { +public: + T* pop() { + if (max_local_elements > 0) { + if (auto* result = local_free_list().pop()) { + return result; + } + } + + guard_ptr guard; + + while (true) { + // (1) - this acquire-load synchronizes-with the release-CAS (3) + guard = acquire_guard(head, std::memory_order_acquire); + if (guard.get() == nullptr) { + return nullptr; + } + + // Note: ref_count can be anything here since multiple threads + // could have gotten a reference to the node on the freelist. + marked_ptr expected(guard); + auto next = guard->next_free().load(std::memory_order_relaxed); + // since head is only changed via CAS operations it is sufficient to use relaxed order + // for this operation as it is always part of a release-sequence headed by (3) + if (head.compare_exchange_weak(expected, next, std::memory_order_relaxed)) { + assert((guard->ref_count().load(std::memory_order_relaxed) & RefCountClaimBit) != 0 && + "ClaimBit must be set for a node on the free list"); + + auto* ptr = guard.get(); + ptr->ref_count().fetch_sub(RefCountClaimBit, std::memory_order_relaxed); // clear claim bit + ptr->next_free().store(nullptr, std::memory_order_relaxed); + guard.ptr.reset(); // reset guard_ptr to prevent decrement of ref_count + return ptr; + } + } + } + + void push(T* node) { + assert(node->ref_count().load(std::memory_order_relaxed) & RefCountClaimBit && + "ClaimBit must be set for a node to be put on the free list"); + if (max_local_elements > 0 && local_free_list().push(node)) { + return; + } + + add_nodes(node, node); + } + +private: + void add_nodes(T* first, T* last) { + // (2) - this acquire-load synchronizes-with the release-CAS (3) + auto old = head.load(std::memory_order_acquire); + do { + last->next_free().store(old, std::memory_order_relaxed); + // (3) - if this release-CAS succeeds, it synchronizes-with the acquire-loads (1, 2) + // if it failes, the reload synchronizes-with itself (3) + } while (!head.compare_exchange_weak(old, first, std::memory_order_release, std::memory_order_acquire)); + } + + // the free list is implemented as a FILO single linked list + // the LSB of a node's ref_count acts as claim bit, so for all nodes on the free list the bit has to be set + concurrent_ptr head; + + class thread_local_free_list { + public: + ~thread_local_free_list() noexcept { + if (head == nullptr) { + return; + } + auto* first = head; + auto* last = head; + auto next = last->next_free().load(std::memory_order_relaxed); + while (next) { + last = next.get(); + next = next->next_free().load(std::memory_order_relaxed); + } + global_free_list.add_nodes(first, last); + } + + bool push(T* node) { + if (number_of_elements >= max_local_elements) { + return false; + } + node->next_free().store(head, std::memory_order_relaxed); + head = node; + ++number_of_elements; + return true; + } + + T* pop() { + auto* result = head; + if (result) { + assert(number_of_elements > 0); + head = result->next_free().load(std::memory_order_relaxed).get(); + --number_of_elements; + // clear claim bit and increment ref_count + result->ref_count().fetch_add(RefCountInc - RefCountClaimBit, std::memory_order_relaxed); + result->next_free().store(nullptr, std::memory_order_relaxed); + } + return result; + } + + private: + size_t number_of_elements = 0; + T* head = nullptr; + }; + + static constexpr size_t max_local_elements = Traits::thread_local_free_list_size; + + static thread_local_free_list& local_free_list() { + // workaround for gcc issue causing redefinition of __tls_guard when + // defining this as static thread_local member of free_list. + alignas(64) static thread_local thread_local_free_list local_free_list; + return local_free_list; + } +}; + +template +template +void* lock_free_ref_count::enable_concurrent_ptr::operator new(size_t sz) { + assert(sz == sizeof(T) && "Cannot handle allocations of anything other than T instances"); + T* result = global_free_list.pop(); + if (result == nullptr) { + auto h = static_cast(::operator new(sz + sizeof(header))); + h->ref_count.store(RefCountInc, std::memory_order_release); + result = static_cast(static_cast(h + 1)); + } + + return result; +} + +template +template +void lock_free_ref_count::enable_concurrent_ptr::operator delete(void* p) { + auto* node = static_cast(p); + assert((node->ref_count().load(std::memory_order_relaxed) & RefCountClaimBit) == 0); + + if (node->decrement_refcnt()) { + node->push_to_free_list(); + } +} + +template +template +bool lock_free_ref_count::enable_concurrent_ptr::decrement_refcnt() { + unsigned old_refcnt, new_refcnt; + do { + old_refcnt = ref_count().load(std::memory_order_relaxed); + new_refcnt = old_refcnt - RefCountInc; + if (new_refcnt == 0) { + new_refcnt = RefCountClaimBit; + } + // (4) - this release/acquire CAS synchronizes with itself + } while (!ref_count().compare_exchange_weak(old_refcnt, + new_refcnt, + new_refcnt == RefCountClaimBit ? std::memory_order_acquire + : std::memory_order_release, + std::memory_order_relaxed)); + + // free node iff ref_count is zero AND we're the first thread to "claim" this node for reclamation. + return ((old_refcnt - new_refcnt) & RefCountClaimBit) != 0; +} + +template +template +lock_free_ref_count::guard_ptr::guard_ptr(const MarkedPtr& p) noexcept : base(p) { + if (this->ptr.get() != nullptr) { + this->ptr->ref_count().fetch_add(RefCountInc, std::memory_order_relaxed); + } +} + +template +template +lock_free_ref_count::guard_ptr::guard_ptr(const guard_ptr& p) noexcept : guard_ptr(p.ptr) {} + +template +template +lock_free_ref_count::guard_ptr::guard_ptr(guard_ptr&& p) noexcept : base(p.ptr) { + p.ptr.reset(); +} + +template +template +auto lock_free_ref_count::guard_ptr::operator=(const guard_ptr& p) -> guard_ptr& { + if (&p == this) { + return *this; + } + + reset(); + this->ptr = p.ptr; + if (this->ptr.get() != nullptr) { + this->ptr->ref_count().fetch_add(RefCountInc, std::memory_order_relaxed); + } + return *this; +} + +template +template +auto lock_free_ref_count::guard_ptr::operator=(guard_ptr&& p) noexcept -> guard_ptr& { + if (&p == this) { + return *this; + } + + reset(); + this->ptr = std::move(p.ptr); + p.ptr.reset(); + return *this; +} + +template +template +void lock_free_ref_count::guard_ptr::acquire(const concurrent_ptr& p, + std::memory_order order) noexcept { + for (;;) { + reset(); + // FIXME: If this load is relaxed, TSan reports a data race between the following + // fetch-add and the initialization of the ref_count field. I tend to disagree, as + // the fetch_add should be ordered after the initial store (operator new) in the + // modification order of ref_count. Therefore the acquire-fetch-add should + // synchronize-with the release store. + // I created a GitHub issue: + // But for now, let's make this an acquire-load to make TSan happy. + auto q = p.load(std::memory_order_acquire); + this->ptr = q; + if (q.get() == nullptr) { + return; + } + + // (5) - this acquire-fetch_add synchronizes-with the release-fetch_sub (7) + // this ensures that a change to p becomes visible + q->ref_count().fetch_add(RefCountInc, std::memory_order_acquire); + + if (q == p.load(order)) { + return; + } + } +} + +template +template +bool lock_free_ref_count::guard_ptr::acquire_if_equal(const concurrent_ptr& p, + const MarkedPtr& expected, + std::memory_order order) noexcept { + reset(); + // FIXME: same issue with TSan as in acquire (see above). + auto q = p.load(std::memory_order_acquire); + if (q != expected) { + return false; + } + + this->ptr = q; + if (q.get() == nullptr) { + return true; + } + + // (6) - this acquire-fetch_add synchronizes-with the release-fetch_sub (7) + // this ensures that a change to p becomes visible + q->ref_count().fetch_add(RefCountInc, std::memory_order_acquire); + + if (q == p.load(order)) { + return true; + } + + reset(); + return false; +} + +template +template +void lock_free_ref_count::guard_ptr::reset() noexcept { + auto* p = this->ptr.get(); + this->ptr.reset(); + if (p == nullptr) { + return; + } + + if (p->decrement_refcnt()) { + if (!p->is_destroyed()) { + p->~T(); + } + + p->push_to_free_list(); + } +} + +template +template +void lock_free_ref_count::guard_ptr::reclaim(Deleter) noexcept { + if (this->ptr.get() != nullptr) { + assert(this->ptr->refs() > 1); + // ref_count was initialized with "1", so we need an additional + // decrement to ensure that the node gets reclaimed. + // ref_count cannot drop to zero here -> no check required. + // (7) - this release-fetch-sub synchronizes-with the acquire-fetch-add (5, 6) + this->ptr->ref_count().fetch_sub(RefCountInc, std::memory_order_release); + } + reset(); +} + +template +template +typename lock_free_ref_count::template enable_concurrent_ptr::free_list + lock_free_ref_count::enable_concurrent_ptr::global_free_list; + +#ifdef TRACK_ALLOCATIONS +template +inline detail::allocation_counter& lock_free_ref_count::allocation_counter() { + return allocation_counter_; +}; + +template +inline void lock_free_ref_count::count_allocation() { + allocation_counter().count_allocation(); +} + +template +inline void lock_free_ref_count::count_reclamation() { + allocation_counter().count_reclamation(); +} +#endif +} // namespace xenium::reclamation + +#ifdef _MSC_VER + #pragma warning(pop) +#endif diff --git a/xenium/reclamation/impl/quiescent_state_based.hpp b/include/xenium/reclamation/impl/quiescent_state_based.hpp similarity index 97% rename from xenium/reclamation/impl/quiescent_state_based.hpp rename to include/xenium/reclamation/impl/quiescent_state_based.hpp index 530cb8d..f4be40c 100644 --- a/xenium/reclamation/impl/quiescent_state_based.hpp +++ b/include/xenium/reclamation/impl/quiescent_state_based.hpp @@ -1,271 +1,271 @@ -// -// Copyright (c) 2018-2020 Manuel Pöter. -// Licensed under the MIT License. See LICENSE file in the project root for full license information. -// - -#ifndef QUIESCENT_STATE_BASED_IMPL - #error "This is an impl file and must not be included directly!" -#endif - -#include -#include - -#include - -namespace xenium::reclamation { - -struct quiescent_state_based::thread_control_block : detail::thread_block_list::entry { - std::atomic local_epoch; -}; - -struct quiescent_state_based::thread_data { - ~thread_data() { - if (control_block == nullptr) { - return; // no control_block -> nothing to do - } - - // we can avoid creating an orphan in case we have no retired nodes left. - if (std::any_of(retire_lists.begin(), retire_lists.end(), [](auto p) { return p != nullptr; })) { - // global_epoch - 1 (mod number_epochs) guarantees a full cycle, making sure no - // other thread may still have a reference to an object in one of the retire lists. - auto target_epoch = (global_epoch.load(std::memory_order_relaxed) + number_epochs - 1) % number_epochs; - assert(target_epoch < number_epochs); - global_thread_block_list.abandon_retired_nodes(new detail::orphan(target_epoch, retire_lists)); - } - - global_thread_block_list.release_entry(control_block); - control_block = nullptr; - } - - void enter_region() { - ensure_has_control_block(); - ++region_entries; - } - - void leave_region() { - if (--region_entries == 0) { - quiescent_state(); - } - } - - void add_retired_node(detail::deletable_object* p) { - assert(control_block != nullptr); - add_retired_node(p, control_block->local_epoch.load(std::memory_order_relaxed)); - } - -private: - void ensure_has_control_block() { - if (control_block == nullptr) { - control_block = global_thread_block_list.acquire_entry(); - auto epoch = global_epoch.load(std::memory_order_relaxed); - do { - control_block->local_epoch.store(epoch, std::memory_order_relaxed); - - // (1) - this acq_rel-CAS synchronizes-with the acquire-load (2) - // and the acq_rel-CAS (5) - } while (!global_epoch.compare_exchange_weak(epoch, epoch, std::memory_order_acq_rel, std::memory_order_relaxed)); - } - } - - void quiescent_state() { - // (2) - this acquire-load synchronizes-with the acq_rel-CAS (1, 5) - auto epoch = global_epoch.load(std::memory_order_acquire); - - if (control_block->local_epoch.load(std::memory_order_relaxed) == epoch) { - const auto new_epoch = (epoch + 1) % number_epochs; - if (!try_update_epoch(epoch, new_epoch)) { - return; - } - - epoch = new_epoch; - } - - // we either just updated the global_epoch or we are observing a new epoch from some other thread - // either way - we can reclaim all the objects from the old 'incarnation' of this epoch - - // (3) - this release-store synchronizes-with the acquire-fence (4) - control_block->local_epoch.store(epoch, std::memory_order_release); - detail::delete_objects(retire_lists[epoch]); - } - - void add_retired_node(detail::deletable_object* p, size_t epoch) { - assert(epoch < number_epochs); - p->next = retire_lists[epoch]; - retire_lists[epoch] = p; - } - - bool try_update_epoch(unsigned curr_epoch, unsigned new_epoch) { - const auto old_epoch = (curr_epoch + number_epochs - 1) % number_epochs; - auto prevents_update = [old_epoch](const thread_control_block& data) { - // TSan does not support explicit fences, so we cannot rely on the acquire-fence (6) - // but have to perform an acquire-load here to avoid false positives. - constexpr auto memory_order = TSAN_MEMORY_ORDER(std::memory_order_acquire, std::memory_order_relaxed); - return data.local_epoch.load(memory_order) == old_epoch && data.is_active(memory_order); - }; - - // If any thread hasn't advanced to the current epoch, abort the attempt. - bool cannot_update = std::any_of(global_thread_block_list.begin(), global_thread_block_list.end(), prevents_update); - if (cannot_update) { - return false; - } - - if (global_epoch.load(std::memory_order_relaxed) == curr_epoch) { - // (4) - this acquire-fence synchronizes-with the release-store (3) - XENIUM_THREAD_FENCE(std::memory_order_acquire); - - // (5) - this acq_rel-CAS synchronizes-with the acquire-load (2) - // and the acq_rel-CAS (1) - bool success = global_epoch.compare_exchange_strong( - curr_epoch, new_epoch, std::memory_order_acq_rel, std::memory_order_relaxed); - if (success) { - adopt_orphans(); - } - } - - // return true regardless of whether the CAS operation was successful or not - // it's not import that THIS thread updated the epoch, but it got updated in any case - return true; - } - - void adopt_orphans() { - auto* cur = global_thread_block_list.adopt_abandoned_retired_nodes(); - for (detail::deletable_object* next = nullptr; cur != nullptr; cur = next) { - next = cur->next; - cur->next = nullptr; - add_retired_node(cur, static_cast*>(cur)->target_epoch); - } - } - - unsigned region_entries = 0; - thread_control_block* control_block = nullptr; - std::array retire_lists = {}; - - friend class quiescent_state_based; - ALLOCATION_COUNTER(quiescent_state_based); -}; - -inline quiescent_state_based::region_guard::region_guard() noexcept { - local_thread_data().enter_region(); -} - -inline quiescent_state_based::region_guard::~region_guard() noexcept { - local_thread_data().leave_region(); -} - -template -quiescent_state_based::guard_ptr::guard_ptr(const MarkedPtr& p) noexcept : base(p) { - if (this->ptr) { - local_thread_data().enter_region(); - } -} - -template -quiescent_state_based::guard_ptr::guard_ptr(const guard_ptr& p) noexcept : guard_ptr(MarkedPtr(p)) {} - -template -quiescent_state_based::guard_ptr::guard_ptr(guard_ptr&& p) noexcept : base(p.ptr) { - p.ptr.reset(); -} - -template -typename quiescent_state_based::template guard_ptr& - quiescent_state_based::guard_ptr::operator=(const guard_ptr& p) noexcept { - if (&p == this) { - return *this; - } - - reset(); - this->ptr = p.ptr; - if (this->ptr) { - local_thread_data().enter_region(); - } - - return *this; -} - -template -typename quiescent_state_based::template guard_ptr& - quiescent_state_based::guard_ptr::operator=(guard_ptr&& p) noexcept { - if (&p == this) { - return *this; - } - - reset(); - this->ptr = std::move(p.ptr); - p.ptr.reset(); - - return *this; -} - -template -void quiescent_state_based::guard_ptr::acquire(const concurrent_ptr& p, - std::memory_order order) noexcept { - if (p.load(std::memory_order_relaxed) == nullptr) { - reset(); - return; - } - - if (!this->ptr) { - local_thread_data().enter_region(); - } - // (6) - this load operation potentially synchronizes-with any release operation on p. - this->ptr = p.load(order); - if (!this->ptr) { - local_thread_data().leave_region(); - } -} - -template -bool quiescent_state_based::guard_ptr::acquire_if_equal(const concurrent_ptr& p, - const MarkedPtr& expected, - std::memory_order order) noexcept { - auto actual = p.load(std::memory_order_relaxed); - if (actual == nullptr || actual != expected) { - reset(); - return actual == expected; - } - - if (!this->ptr) { - local_thread_data().enter_region(); - } - // (7) - this load operation potentially synchronizes-with any release operation on p. - this->ptr = p.load(order); - if (!this->ptr || this->ptr != expected) { - local_thread_data().leave_region(); - this->ptr.reset(); - } - - return this->ptr == expected; -} - -template -void quiescent_state_based::guard_ptr::reset() noexcept { - if (this->ptr) { - local_thread_data().leave_region(); - } - this->ptr.reset(); -} - -template -void quiescent_state_based::guard_ptr::reclaim(Deleter d) noexcept { - this->ptr->set_deleter(std::move(d)); - local_thread_data().add_retired_node(this->ptr.get()); - reset(); -} - -inline quiescent_state_based::thread_data& quiescent_state_based::local_thread_data() { - // workaround for a GCC issue with multiple definitions of __tls_guard - static thread_local thread_data local_thread_data; - return local_thread_data; -} - -#ifdef TRACK_ALLOCATIONS -inline void quiescent_state_based::count_allocation() { - local_thread_data().allocation_counter.count_allocation(); -} - -inline void quiescent_state_based::count_reclamation() { - local_thread_data().allocation_counter.count_reclamation(); -} -#endif -} // namespace xenium::reclamation +// +// Copyright (c) 2018-2020 Manuel Pöter. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +#ifndef QUIESCENT_STATE_BASED_IMPL + #error "This is an impl file and must not be included directly!" +#endif + +#include +#include + +#include + +namespace xenium::reclamation { + +struct quiescent_state_based::thread_control_block : detail::thread_block_list::entry { + std::atomic local_epoch; +}; + +struct quiescent_state_based::thread_data { + ~thread_data() { + if (control_block == nullptr) { + return; // no control_block -> nothing to do + } + + // we can avoid creating an orphan in case we have no retired nodes left. + if (std::any_of(retire_lists.begin(), retire_lists.end(), [](auto p) { return p != nullptr; })) { + // global_epoch - 1 (mod number_epochs) guarantees a full cycle, making sure no + // other thread may still have a reference to an object in one of the retire lists. + auto target_epoch = (global_epoch.load(std::memory_order_relaxed) + number_epochs - 1) % number_epochs; + assert(target_epoch < number_epochs); + global_thread_block_list.abandon_retired_nodes(new detail::orphan(target_epoch, retire_lists)); + } + + global_thread_block_list.release_entry(control_block); + control_block = nullptr; + } + + void enter_region() { + ensure_has_control_block(); + ++region_entries; + } + + void leave_region() { + if (--region_entries == 0) { + quiescent_state(); + } + } + + void add_retired_node(detail::deletable_object* p) { + assert(control_block != nullptr); + add_retired_node(p, control_block->local_epoch.load(std::memory_order_relaxed)); + } + +private: + void ensure_has_control_block() { + if (control_block == nullptr) { + control_block = global_thread_block_list.acquire_entry(); + auto epoch = global_epoch.load(std::memory_order_relaxed); + do { + control_block->local_epoch.store(epoch, std::memory_order_relaxed); + + // (1) - this acq_rel-CAS synchronizes-with the acquire-load (2) + // and the acq_rel-CAS (5) + } while (!global_epoch.compare_exchange_weak(epoch, epoch, std::memory_order_acq_rel, std::memory_order_relaxed)); + } + } + + void quiescent_state() { + // (2) - this acquire-load synchronizes-with the acq_rel-CAS (1, 5) + auto epoch = global_epoch.load(std::memory_order_acquire); + + if (control_block->local_epoch.load(std::memory_order_relaxed) == epoch) { + const auto new_epoch = (epoch + 1) % number_epochs; + if (!try_update_epoch(epoch, new_epoch)) { + return; + } + + epoch = new_epoch; + } + + // we either just updated the global_epoch or we are observing a new epoch from some other thread + // either way - we can reclaim all the objects from the old 'incarnation' of this epoch + + // (3) - this release-store synchronizes-with the acquire-fence (4) + control_block->local_epoch.store(epoch, std::memory_order_release); + detail::delete_objects(retire_lists[epoch]); + } + + void add_retired_node(detail::deletable_object* p, size_t epoch) { + assert(epoch < number_epochs); + p->next = retire_lists[epoch]; + retire_lists[epoch] = p; + } + + bool try_update_epoch(unsigned curr_epoch, unsigned new_epoch) { + const auto old_epoch = (curr_epoch + number_epochs - 1) % number_epochs; + auto prevents_update = [old_epoch](const thread_control_block& data) { + // TSan does not support explicit fences, so we cannot rely on the acquire-fence (6) + // but have to perform an acquire-load here to avoid false positives. + constexpr auto memory_order = TSAN_MEMORY_ORDER(std::memory_order_acquire, std::memory_order_relaxed); + return data.local_epoch.load(memory_order) == old_epoch && data.is_active(memory_order); + }; + + // If any thread hasn't advanced to the current epoch, abort the attempt. + bool cannot_update = std::any_of(global_thread_block_list.begin(), global_thread_block_list.end(), prevents_update); + if (cannot_update) { + return false; + } + + if (global_epoch.load(std::memory_order_relaxed) == curr_epoch) { + // (4) - this acquire-fence synchronizes-with the release-store (3) + XENIUM_THREAD_FENCE(std::memory_order_acquire); + + // (5) - this acq_rel-CAS synchronizes-with the acquire-load (2) + // and the acq_rel-CAS (1) + bool success = global_epoch.compare_exchange_strong( + curr_epoch, new_epoch, std::memory_order_acq_rel, std::memory_order_relaxed); + if (success) { + adopt_orphans(); + } + } + + // return true regardless of whether the CAS operation was successful or not + // it's not import that THIS thread updated the epoch, but it got updated in any case + return true; + } + + void adopt_orphans() { + auto* cur = global_thread_block_list.adopt_abandoned_retired_nodes(); + for (detail::deletable_object* next = nullptr; cur != nullptr; cur = next) { + next = cur->next; + cur->next = nullptr; + add_retired_node(cur, static_cast*>(cur)->target_epoch); + } + } + + unsigned region_entries = 0; + thread_control_block* control_block = nullptr; + std::array retire_lists = {}; + + friend class quiescent_state_based; + ALLOCATION_COUNTER(quiescent_state_based); +}; + +inline quiescent_state_based::region_guard::region_guard() noexcept { + local_thread_data().enter_region(); +} + +inline quiescent_state_based::region_guard::~region_guard() noexcept { + local_thread_data().leave_region(); +} + +template +quiescent_state_based::guard_ptr::guard_ptr(const MarkedPtr& p) noexcept : base(p) { + if (this->ptr) { + local_thread_data().enter_region(); + } +} + +template +quiescent_state_based::guard_ptr::guard_ptr(const guard_ptr& p) noexcept : guard_ptr(MarkedPtr(p)) {} + +template +quiescent_state_based::guard_ptr::guard_ptr(guard_ptr&& p) noexcept : base(p.ptr) { + p.ptr.reset(); +} + +template +typename quiescent_state_based::template guard_ptr& + quiescent_state_based::guard_ptr::operator=(const guard_ptr& p) noexcept { + if (&p == this) { + return *this; + } + + reset(); + this->ptr = p.ptr; + if (this->ptr) { + local_thread_data().enter_region(); + } + + return *this; +} + +template +typename quiescent_state_based::template guard_ptr& + quiescent_state_based::guard_ptr::operator=(guard_ptr&& p) noexcept { + if (&p == this) { + return *this; + } + + reset(); + this->ptr = std::move(p.ptr); + p.ptr.reset(); + + return *this; +} + +template +void quiescent_state_based::guard_ptr::acquire(const concurrent_ptr& p, + std::memory_order order) noexcept { + if (p.load(std::memory_order_relaxed) == nullptr) { + reset(); + return; + } + + if (!this->ptr) { + local_thread_data().enter_region(); + } + // (6) - this load operation potentially synchronizes-with any release operation on p. + this->ptr = p.load(order); + if (!this->ptr) { + local_thread_data().leave_region(); + } +} + +template +bool quiescent_state_based::guard_ptr::acquire_if_equal(const concurrent_ptr& p, + const MarkedPtr& expected, + std::memory_order order) noexcept { + auto actual = p.load(std::memory_order_relaxed); + if (actual == nullptr || actual != expected) { + reset(); + return actual == expected; + } + + if (!this->ptr) { + local_thread_data().enter_region(); + } + // (7) - this load operation potentially synchronizes-with any release operation on p. + this->ptr = p.load(order); + if (!this->ptr || this->ptr != expected) { + local_thread_data().leave_region(); + this->ptr.reset(); + } + + return this->ptr == expected; +} + +template +void quiescent_state_based::guard_ptr::reset() noexcept { + if (this->ptr) { + local_thread_data().leave_region(); + } + this->ptr.reset(); +} + +template +void quiescent_state_based::guard_ptr::reclaim(Deleter d) noexcept { + this->ptr->set_deleter(std::move(d)); + local_thread_data().add_retired_node(this->ptr.get()); + reset(); +} + +inline quiescent_state_based::thread_data& quiescent_state_based::local_thread_data() { + // workaround for a GCC issue with multiple definitions of __tls_guard + static thread_local thread_data local_thread_data; + return local_thread_data; +} + +#ifdef TRACK_ALLOCATIONS +inline void quiescent_state_based::count_allocation() { + local_thread_data().allocation_counter.count_allocation(); +} + +inline void quiescent_state_based::count_reclamation() { + local_thread_data().allocation_counter.count_reclamation(); +} +#endif +} // namespace xenium::reclamation diff --git a/xenium/reclamation/impl/stamp_it.hpp b/include/xenium/reclamation/impl/stamp_it.hpp similarity index 100% rename from xenium/reclamation/impl/stamp_it.hpp rename to include/xenium/reclamation/impl/stamp_it.hpp diff --git a/xenium/reclamation/lock_free_ref_count.hpp b/include/xenium/reclamation/lock_free_ref_count.hpp similarity index 97% rename from xenium/reclamation/lock_free_ref_count.hpp rename to include/xenium/reclamation/lock_free_ref_count.hpp index 18a5866..2377cac 100644 --- a/xenium/reclamation/lock_free_ref_count.hpp +++ b/include/xenium/reclamation/lock_free_ref_count.hpp @@ -1,207 +1,207 @@ -// -// Copyright (c) 2018-2020 Manuel Pöter. -// Licensed under the MIT License. See LICENSE file in the project root for full license information. -// - -#ifndef XENIUM_LOCK_FREE_REF_COUNT_HPP -#define XENIUM_LOCK_FREE_REF_COUNT_HPP - -#include -#include -#include - -#include -#include - -#include - -namespace xenium { - -namespace policy { - /** - * @brief Policy to configure whether to insert padding after the internal header for - * `lock_free_ref_count` reclamation. - * - * This policy is used to define whether a padding should be inserted between the internal - * header that contains the reference counter and the actual object. This can be used to - * avoid false sharing, but of course it increases memory overhead, which can also cause - * a performance drop in some cases. - * - * @tparam Value - */ - template - struct insert_padding; - - /** - * @brief Policy to configure the size of thread-local free-lists for `lock` reclamation. - * - * This policy is used to define the max. number of items in each thread-local free-list. - * If this is set to zero, then thread-local free-lists are completely disable. - * Using thread-local free-lists cna reduce the contention on the global free-list, but it - * may lead to increased memory usage, since items stored in a thread-local free-list can - * only be reused by the owning thread. - * - * @tparam Value - */ - template - struct thread_local_free_list_size; -} // namespace policy - -namespace reclamation { - template - struct lock_free_ref_count_traits { - static constexpr bool insert_padding = InsertPadding; - static constexpr std::size_t thread_local_free_list_size = ThreadLocalFreeListSize; - - template - using with = lock_free_ref_count_traits< - parameter::value_param_t::value, - parameter::value_param_t:: - value>; - }; - - /** - * @brief An implementation of the lock-free reference counting (LFRC) schemea as proposed - * by Valois \[[Val95](index.html#ref-valois-1995), [MS95](index.html#ref-michael-1995)\]. - * - * This scheme cannot handle types that define their own new/delete operators, and it - * does not allow the use of custom deleters. - * - * This class does not take a list of policies, but a `Traits` type that can be customized - * with a list of policies. The following policies are supported: - * * `xenium::policy::insert_padding`
- * Defines whether a padding should be inserted between the internal header and the actual - * object. (defaults to false) - * * `xenium::policy::thread_local_free_list_size`
- * Defines the max. number of items in each thread-local free-list. (defaults to 0) - * - * @tparam Traits - */ - template > - class lock_free_ref_count { - template - class guard_ptr; - - public: - template - using with = lock_free_ref_count>; - - template - using concurrent_ptr = detail::concurrent_ptr; - - template > - class enable_concurrent_ptr; - - class region_guard {}; - - ALLOCATION_TRACKER - private: - static constexpr unsigned RefCountInc = 2; - static constexpr unsigned RefCountClaimBit = 1; - - ALLOCATION_TRACKING_FUNCTIONS; -#ifdef TRACK_ALLOCATIONS - inline static thread_local detail::registered_allocation_counter allocation_counter_; - static detail::allocation_counter& allocation_counter(); -#endif - }; - - template - template - class lock_free_ref_count::enable_concurrent_ptr : private detail::tracked_object { - public: - enable_concurrent_ptr(const enable_concurrent_ptr&) noexcept = delete; - enable_concurrent_ptr(enable_concurrent_ptr&&) noexcept = delete; - enable_concurrent_ptr& operator=(const enable_concurrent_ptr&) noexcept = delete; - enable_concurrent_ptr& operator=(enable_concurrent_ptr&&) noexcept = delete; - - protected: - enable_concurrent_ptr() noexcept { destroyed().store(false, std::memory_order_relaxed); } - virtual ~enable_concurrent_ptr() noexcept { - assert(!is_destroyed()); - destroyed().store(true, std::memory_order_relaxed); - } - - public: - using Deleter = DeleterT; - static_assert(std::is_same>::value, - "lock_free_ref_count reclamation can only be used with std::default_delete as Deleter."); - - static constexpr std::size_t number_of_mark_bits = N; - [[nodiscard]] unsigned refs() const { return getHeader()->ref_count.load(std::memory_order_relaxed) >> 1; } - - void* operator new(size_t sz); - void operator delete(void* p); - - private: - bool decrement_refcnt(); - [[nodiscard]] bool is_destroyed() const { return getHeader()->destroyed.load(std::memory_order_relaxed); } - void push_to_free_list() { global_free_list.push(static_cast(this)); } - - struct unpadded_header { - std::atomic ref_count; - std::atomic destroyed; - concurrent_ptr next_free; - }; - struct padded_header : unpadded_header { - char padding[64 - sizeof(unpadded_header)]; - }; - using header = std::conditional_t; - header* getHeader() { return static_cast(static_cast(this)) - 1; } - [[nodiscard]] const header* getHeader() const { - return static_cast(static_cast(this)) - 1; - } - - std::atomic& ref_count() { return getHeader()->ref_count; } - std::atomic& destroyed() { return getHeader()->destroyed; } - concurrent_ptr& next_free() { return getHeader()->next_free; } - - friend class lock_free_ref_count; - - using guard_ptr = typename concurrent_ptr::guard_ptr; - using marked_ptr = typename concurrent_ptr::marked_ptr; - - class free_list; - static free_list global_free_list; - }; - - template - template - class lock_free_ref_count::guard_ptr : public detail::guard_ptr> { - using base = detail::guard_ptr; - using Deleter = typename T::Deleter; - - public: - template - friend class enable_concurrent_ptr; - - // Guard a marked ptr. - explicit guard_ptr(const MarkedPtr& p = MarkedPtr()) noexcept; - guard_ptr(const guard_ptr& p) noexcept; - guard_ptr(guard_ptr&& p) noexcept; - - guard_ptr& operator=(const guard_ptr& p); - guard_ptr& operator=(guard_ptr&& p) noexcept; - - // Atomically take snapshot of p, and *if* it points to unreclaimed object, acquire shared ownership of it. - void acquire(const concurrent_ptr& p, std::memory_order order = std::memory_order_seq_cst) noexcept; - - // Like acquire, but quit early if a snapshot != expected. - bool acquire_if_equal(const concurrent_ptr& p, - const MarkedPtr& expected, - std::memory_order order = std::memory_order_seq_cst) noexcept; - - // Release ownership. Postcondition: get() == nullptr. - void reset() noexcept; - - // Reset. Deleter d will be applied some time after all owners release their ownership. - void reclaim(Deleter d = Deleter()) noexcept; - }; -} // namespace reclamation -} // namespace xenium - -#define LOCK_FREE_REF_COUNT_IMPL -#include -#undef LOCK_FREE_REF_COUNT_IMPL - -#endif +// +// Copyright (c) 2018-2020 Manuel Pöter. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +#ifndef XENIUM_LOCK_FREE_REF_COUNT_HPP +#define XENIUM_LOCK_FREE_REF_COUNT_HPP + +#include +#include +#include + +#include +#include + +#include + +namespace xenium { + +namespace policy { + /** + * @brief Policy to configure whether to insert padding after the internal header for + * `lock_free_ref_count` reclamation. + * + * This policy is used to define whether a padding should be inserted between the internal + * header that contains the reference counter and the actual object. This can be used to + * avoid false sharing, but of course it increases memory overhead, which can also cause + * a performance drop in some cases. + * + * @tparam Value + */ + template + struct insert_padding; + + /** + * @brief Policy to configure the size of thread-local free-lists for `lock` reclamation. + * + * This policy is used to define the max. number of items in each thread-local free-list. + * If this is set to zero, then thread-local free-lists are completely disable. + * Using thread-local free-lists cna reduce the contention on the global free-list, but it + * may lead to increased memory usage, since items stored in a thread-local free-list can + * only be reused by the owning thread. + * + * @tparam Value + */ + template + struct thread_local_free_list_size; +} // namespace policy + +namespace reclamation { + template + struct lock_free_ref_count_traits { + static constexpr bool insert_padding = InsertPadding; + static constexpr std::size_t thread_local_free_list_size = ThreadLocalFreeListSize; + + template + using with = lock_free_ref_count_traits< + parameter::value_param_t::value, + parameter::value_param_t:: + value>; + }; + + /** + * @brief An implementation of the lock-free reference counting (LFRC) schemea as proposed + * by Valois \[[Val95](index.html#ref-valois-1995), [MS95](index.html#ref-michael-1995)\]. + * + * This scheme cannot handle types that define their own new/delete operators, and it + * does not allow the use of custom deleters. + * + * This class does not take a list of policies, but a `Traits` type that can be customized + * with a list of policies. The following policies are supported: + * * `xenium::policy::insert_padding`
+ * Defines whether a padding should be inserted between the internal header and the actual + * object. (defaults to false) + * * `xenium::policy::thread_local_free_list_size`
+ * Defines the max. number of items in each thread-local free-list. (defaults to 0) + * + * @tparam Traits + */ + template > + class lock_free_ref_count { + template + class guard_ptr; + + public: + template + using with = lock_free_ref_count>; + + template + using concurrent_ptr = detail::concurrent_ptr; + + template > + class enable_concurrent_ptr; + + class region_guard {}; + + ALLOCATION_TRACKER + private: + static constexpr unsigned RefCountInc = 2; + static constexpr unsigned RefCountClaimBit = 1; + + ALLOCATION_TRACKING_FUNCTIONS; +#ifdef TRACK_ALLOCATIONS + inline static thread_local detail::registered_allocation_counter allocation_counter_; + static detail::allocation_counter& allocation_counter(); +#endif + }; + + template + template + class lock_free_ref_count::enable_concurrent_ptr : private detail::tracked_object { + public: + enable_concurrent_ptr(const enable_concurrent_ptr&) noexcept = delete; + enable_concurrent_ptr(enable_concurrent_ptr&&) noexcept = delete; + enable_concurrent_ptr& operator=(const enable_concurrent_ptr&) noexcept = delete; + enable_concurrent_ptr& operator=(enable_concurrent_ptr&&) noexcept = delete; + + protected: + enable_concurrent_ptr() noexcept { destroyed().store(false, std::memory_order_relaxed); } + virtual ~enable_concurrent_ptr() noexcept { + assert(!is_destroyed()); + destroyed().store(true, std::memory_order_relaxed); + } + + public: + using Deleter = DeleterT; + static_assert(std::is_same>::value, + "lock_free_ref_count reclamation can only be used with std::default_delete as Deleter."); + + static constexpr std::size_t number_of_mark_bits = N; + [[nodiscard]] unsigned refs() const { return getHeader()->ref_count.load(std::memory_order_relaxed) >> 1; } + + void* operator new(size_t sz); + void operator delete(void* p); + + private: + bool decrement_refcnt(); + [[nodiscard]] bool is_destroyed() const { return getHeader()->destroyed.load(std::memory_order_relaxed); } + void push_to_free_list() { global_free_list.push(static_cast(this)); } + + struct unpadded_header { + std::atomic ref_count; + std::atomic destroyed; + concurrent_ptr next_free; + }; + struct padded_header : unpadded_header { + char padding[64 - sizeof(unpadded_header)]; + }; + using header = std::conditional_t; + header* getHeader() { return static_cast(static_cast(this)) - 1; } + [[nodiscard]] const header* getHeader() const { + return static_cast(static_cast(this)) - 1; + } + + std::atomic& ref_count() { return getHeader()->ref_count; } + std::atomic& destroyed() { return getHeader()->destroyed; } + concurrent_ptr& next_free() { return getHeader()->next_free; } + + friend class lock_free_ref_count; + + using guard_ptr = typename concurrent_ptr::guard_ptr; + using marked_ptr = typename concurrent_ptr::marked_ptr; + + class free_list; + static free_list global_free_list; + }; + + template + template + class lock_free_ref_count::guard_ptr : public detail::guard_ptr> { + using base = detail::guard_ptr; + using Deleter = typename T::Deleter; + + public: + template + friend class enable_concurrent_ptr; + + // Guard a marked ptr. + explicit guard_ptr(const MarkedPtr& p = MarkedPtr()) noexcept; + guard_ptr(const guard_ptr& p) noexcept; + guard_ptr(guard_ptr&& p) noexcept; + + guard_ptr& operator=(const guard_ptr& p); + guard_ptr& operator=(guard_ptr&& p) noexcept; + + // Atomically take snapshot of p, and *if* it points to unreclaimed object, acquire shared ownership of it. + void acquire(const concurrent_ptr& p, std::memory_order order = std::memory_order_seq_cst) noexcept; + + // Like acquire, but quit early if a snapshot != expected. + bool acquire_if_equal(const concurrent_ptr& p, + const MarkedPtr& expected, + std::memory_order order = std::memory_order_seq_cst) noexcept; + + // Release ownership. Postcondition: get() == nullptr. + void reset() noexcept; + + // Reset. Deleter d will be applied some time after all owners release their ownership. + void reclaim(Deleter d = Deleter()) noexcept; + }; +} // namespace reclamation +} // namespace xenium + +#define LOCK_FREE_REF_COUNT_IMPL +#include +#undef LOCK_FREE_REF_COUNT_IMPL + +#endif diff --git a/xenium/reclamation/quiescent_state_based.hpp b/include/xenium/reclamation/quiescent_state_based.hpp similarity index 97% rename from xenium/reclamation/quiescent_state_based.hpp rename to include/xenium/reclamation/quiescent_state_based.hpp index f3738e5..4af1d20 100644 --- a/xenium/reclamation/quiescent_state_based.hpp +++ b/include/xenium/reclamation/quiescent_state_based.hpp @@ -1,114 +1,114 @@ -// -// Copyright (c) 2018-2020 Manuel Pöter. -// Licensed under the MIT License. See LICENSE file in the project root for full license information. -// - -#ifndef XENIUM_QUIESCENT_STATE_BASED_HPP -#define XENIUM_QUIESCENT_STATE_BASED_HPP - -#include -#include -#include -#include -#include - -#include - -namespace xenium::reclamation { - -/** - * @brief Quiescent state based reclamation - */ -class quiescent_state_based { - template - class guard_ptr; - -public: - template > - class enable_concurrent_ptr; - - struct region_guard { - region_guard() noexcept; - ~region_guard() noexcept; - - region_guard(const region_guard&) = delete; - region_guard(region_guard&&) = delete; - region_guard& operator=(const region_guard&) = delete; - region_guard& operator=(region_guard&&) = delete; - }; - - template - using concurrent_ptr = detail::concurrent_ptr; - - ALLOCATION_TRACKER; - -private: - static constexpr unsigned number_epochs = 3; - - struct thread_data; - struct thread_control_block; - - inline static std::atomic global_epoch; - inline static detail::thread_block_list global_thread_block_list; - static thread_data& local_thread_data(); - - ALLOCATION_TRACKING_FUNCTIONS; -}; - -template -class quiescent_state_based::enable_concurrent_ptr : - private detail::deletable_object_impl, - private detail::tracked_object { -public: - static constexpr std::size_t number_of_mark_bits = N; - -protected: - enable_concurrent_ptr() noexcept = default; - enable_concurrent_ptr(const enable_concurrent_ptr&) noexcept = default; - enable_concurrent_ptr(enable_concurrent_ptr&&) noexcept = default; - enable_concurrent_ptr& operator=(const enable_concurrent_ptr&) noexcept = default; - enable_concurrent_ptr& operator=(enable_concurrent_ptr&&) noexcept = default; - ~enable_concurrent_ptr() noexcept override = default; - -private: - friend detail::deletable_object_impl; - - template - friend class guard_ptr; -}; - -template -class quiescent_state_based::guard_ptr : public detail::guard_ptr> { - using base = detail::guard_ptr; - using Deleter = typename T::Deleter; - -public: - // Guard a marked ptr. - explicit guard_ptr(const MarkedPtr& p = MarkedPtr()) noexcept; - guard_ptr(const guard_ptr& p) noexcept; - guard_ptr(guard_ptr&& p) noexcept; - - guard_ptr& operator=(const guard_ptr& p) noexcept; - guard_ptr& operator=(guard_ptr&& p) noexcept; - - // Atomically take snapshot of p, and *if* it points to unreclaimed object, acquire shared ownership of it. - void acquire(const concurrent_ptr& p, std::memory_order order = std::memory_order_seq_cst) noexcept; - - // Like acquire, but quit early if a snapshot != expected. - bool acquire_if_equal(const concurrent_ptr& p, - const MarkedPtr& expected, - std::memory_order order = std::memory_order_seq_cst) noexcept; - - // Release ownership. Postcondition: get() == nullptr. - void reset() noexcept; - - // Reset. Deleter d will be applied some time after all owners release their ownership. - void reclaim(Deleter d = Deleter()) noexcept; -}; -} // namespace xenium::reclamation - -#define QUIESCENT_STATE_BASED_IMPL -#include -#undef QUIESCENT_STATE_BASED_IMPL - +// +// Copyright (c) 2018-2020 Manuel Pöter. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +#ifndef XENIUM_QUIESCENT_STATE_BASED_HPP +#define XENIUM_QUIESCENT_STATE_BASED_HPP + +#include +#include +#include +#include +#include + +#include + +namespace xenium::reclamation { + +/** + * @brief Quiescent state based reclamation + */ +class quiescent_state_based { + template + class guard_ptr; + +public: + template > + class enable_concurrent_ptr; + + struct region_guard { + region_guard() noexcept; + ~region_guard() noexcept; + + region_guard(const region_guard&) = delete; + region_guard(region_guard&&) = delete; + region_guard& operator=(const region_guard&) = delete; + region_guard& operator=(region_guard&&) = delete; + }; + + template + using concurrent_ptr = detail::concurrent_ptr; + + ALLOCATION_TRACKER; + +private: + static constexpr unsigned number_epochs = 3; + + struct thread_data; + struct thread_control_block; + + inline static std::atomic global_epoch; + inline static detail::thread_block_list global_thread_block_list; + static thread_data& local_thread_data(); + + ALLOCATION_TRACKING_FUNCTIONS; +}; + +template +class quiescent_state_based::enable_concurrent_ptr : + private detail::deletable_object_impl, + private detail::tracked_object { +public: + static constexpr std::size_t number_of_mark_bits = N; + +protected: + enable_concurrent_ptr() noexcept = default; + enable_concurrent_ptr(const enable_concurrent_ptr&) noexcept = default; + enable_concurrent_ptr(enable_concurrent_ptr&&) noexcept = default; + enable_concurrent_ptr& operator=(const enable_concurrent_ptr&) noexcept = default; + enable_concurrent_ptr& operator=(enable_concurrent_ptr&&) noexcept = default; + ~enable_concurrent_ptr() noexcept override = default; + +private: + friend detail::deletable_object_impl; + + template + friend class guard_ptr; +}; + +template +class quiescent_state_based::guard_ptr : public detail::guard_ptr> { + using base = detail::guard_ptr; + using Deleter = typename T::Deleter; + +public: + // Guard a marked ptr. + explicit guard_ptr(const MarkedPtr& p = MarkedPtr()) noexcept; + guard_ptr(const guard_ptr& p) noexcept; + guard_ptr(guard_ptr&& p) noexcept; + + guard_ptr& operator=(const guard_ptr& p) noexcept; + guard_ptr& operator=(guard_ptr&& p) noexcept; + + // Atomically take snapshot of p, and *if* it points to unreclaimed object, acquire shared ownership of it. + void acquire(const concurrent_ptr& p, std::memory_order order = std::memory_order_seq_cst) noexcept; + + // Like acquire, but quit early if a snapshot != expected. + bool acquire_if_equal(const concurrent_ptr& p, + const MarkedPtr& expected, + std::memory_order order = std::memory_order_seq_cst) noexcept; + + // Release ownership. Postcondition: get() == nullptr. + void reset() noexcept; + + // Reset. Deleter d will be applied some time after all owners release their ownership. + void reclaim(Deleter d = Deleter()) noexcept; +}; +} // namespace xenium::reclamation + +#define QUIESCENT_STATE_BASED_IMPL +#include +#undef QUIESCENT_STATE_BASED_IMPL + #endif \ No newline at end of file diff --git a/xenium/reclamation/stamp_it.hpp b/include/xenium/reclamation/stamp_it.hpp similarity index 96% rename from xenium/reclamation/stamp_it.hpp rename to include/xenium/reclamation/stamp_it.hpp index c63ec6e..80981b2 100644 --- a/xenium/reclamation/stamp_it.hpp +++ b/include/xenium/reclamation/stamp_it.hpp @@ -1,146 +1,146 @@ -// -// Copyright (c) 2018-2020 Manuel Pöter. -// Licensed under the MIT License. See LICENSE file in the project root for full license information. -// - -#ifndef XENIUM_STAMP_IT_HPP -#define XENIUM_STAMP_IT_HPP - -#include -#include -#include -#include -#include - -#include - -namespace xenium::reclamation { - -/** - * @brief Stamp-it - */ -class stamp_it { - template - class guard_ptr; - -public: - template > - class enable_concurrent_ptr; - - struct region_guard { - region_guard() noexcept; - ~region_guard(); - - region_guard(const region_guard&) = delete; - region_guard(region_guard&&) = delete; - region_guard& operator=(const region_guard&) = delete; - region_guard& operator=(region_guard&&) = delete; - }; - - template - using concurrent_ptr = detail::concurrent_ptr; - -#ifdef WITH_PERF_COUNTER - struct performance_counters { - size_t push_calls = 0; - size_t push_iterations = 0; - size_t remove_calls = 0; - size_t remove_next_iterations = 0; - size_t remove_prev_iterations = 0; - }; - static performance_counters get_performance_counters(); -#endif - - ALLOCATION_TRACKER; - -private: - static constexpr size_t MarkBits = 18; - - using stamp_t = size_t; - - struct deletable_object_with_stamp; - struct thread_control_block; - struct thread_data; - - class thread_order_queue; - - static constexpr stamp_t NotInList = 1; - static constexpr stamp_t PendingPush = 2; - static constexpr stamp_t StampInc = 4; - - static thread_order_queue queue; - static thread_data& local_thread_data(); - - ALLOCATION_TRACKING_FUNCTIONS; -}; - -struct stamp_it::deletable_object_with_stamp { - virtual void delete_self() = 0; - deletable_object_with_stamp* next = nullptr; - deletable_object_with_stamp* next_chunk = nullptr; - -protected: - virtual ~deletable_object_with_stamp() = default; - -private: - stamp_t stamp{}; - friend class stamp_it; -}; - -template -class stamp_it::enable_concurrent_ptr : - private detail::deletable_object_impl, - private detail::tracked_object { -public: - static constexpr std::size_t number_of_mark_bits = N; - -protected: - enable_concurrent_ptr() noexcept = default; - enable_concurrent_ptr(const enable_concurrent_ptr&) noexcept = default; - enable_concurrent_ptr(enable_concurrent_ptr&&) noexcept = default; - enable_concurrent_ptr& operator=(const enable_concurrent_ptr&) noexcept = default; - enable_concurrent_ptr& operator=(enable_concurrent_ptr&&) noexcept = default; - ~enable_concurrent_ptr() noexcept override = default; - -private: - friend detail::deletable_object_impl; - - template - friend class guard_ptr; -}; - -template -class stamp_it::guard_ptr : public detail::guard_ptr> { - using base = detail::guard_ptr; - using Deleter = typename T::Deleter; - -public: - // Guard a marked ptr. - explicit guard_ptr(const MarkedPtr& p = MarkedPtr()) noexcept; - guard_ptr(const guard_ptr& p) noexcept; - guard_ptr(guard_ptr&& p) noexcept; - - guard_ptr& operator=(const guard_ptr& p) noexcept; - guard_ptr& operator=(guard_ptr&& p) noexcept; - - // Atomically take snapshot of p, and *if* it points to unreclaimed object, acquire shared ownership of it. - void acquire(const concurrent_ptr& p, std::memory_order order = std::memory_order_seq_cst) noexcept; - - // Like acquire, but quit early if a snapshot != expected. - bool acquire_if_equal(const concurrent_ptr& p, - const MarkedPtr& expected, - std::memory_order order = std::memory_order_seq_cst) noexcept; - - // Release ownership. Postcondition: get() == nullptr. - void reset() noexcept; - - // Reset. Deleter d will be applied some time after all owners release their ownership. - void reclaim(Deleter d = Deleter()) noexcept; -}; -} // namespace xenium::reclamation - -#define STAMP_IT_IMPL -#include -#undef STAMP_IT_IMPL - -#endif +// +// Copyright (c) 2018-2020 Manuel Pöter. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +#ifndef XENIUM_STAMP_IT_HPP +#define XENIUM_STAMP_IT_HPP + +#include +#include +#include +#include +#include + +#include + +namespace xenium::reclamation { + +/** + * @brief Stamp-it + */ +class stamp_it { + template + class guard_ptr; + +public: + template > + class enable_concurrent_ptr; + + struct region_guard { + region_guard() noexcept; + ~region_guard(); + + region_guard(const region_guard&) = delete; + region_guard(region_guard&&) = delete; + region_guard& operator=(const region_guard&) = delete; + region_guard& operator=(region_guard&&) = delete; + }; + + template + using concurrent_ptr = detail::concurrent_ptr; + +#ifdef WITH_PERF_COUNTER + struct performance_counters { + size_t push_calls = 0; + size_t push_iterations = 0; + size_t remove_calls = 0; + size_t remove_next_iterations = 0; + size_t remove_prev_iterations = 0; + }; + static performance_counters get_performance_counters(); +#endif + + ALLOCATION_TRACKER; + +private: + static constexpr size_t MarkBits = 18; + + using stamp_t = size_t; + + struct deletable_object_with_stamp; + struct thread_control_block; + struct thread_data; + + class thread_order_queue; + + static constexpr stamp_t NotInList = 1; + static constexpr stamp_t PendingPush = 2; + static constexpr stamp_t StampInc = 4; + + static thread_order_queue queue; + static thread_data& local_thread_data(); + + ALLOCATION_TRACKING_FUNCTIONS; +}; + +struct stamp_it::deletable_object_with_stamp { + virtual void delete_self() = 0; + deletable_object_with_stamp* next = nullptr; + deletable_object_with_stamp* next_chunk = nullptr; + +protected: + virtual ~deletable_object_with_stamp() = default; + +private: + stamp_t stamp{}; + friend class stamp_it; +}; + +template +class stamp_it::enable_concurrent_ptr : + private detail::deletable_object_impl, + private detail::tracked_object { +public: + static constexpr std::size_t number_of_mark_bits = N; + +protected: + enable_concurrent_ptr() noexcept = default; + enable_concurrent_ptr(const enable_concurrent_ptr&) noexcept = default; + enable_concurrent_ptr(enable_concurrent_ptr&&) noexcept = default; + enable_concurrent_ptr& operator=(const enable_concurrent_ptr&) noexcept = default; + enable_concurrent_ptr& operator=(enable_concurrent_ptr&&) noexcept = default; + ~enable_concurrent_ptr() noexcept override = default; + +private: + friend detail::deletable_object_impl; + + template + friend class guard_ptr; +}; + +template +class stamp_it::guard_ptr : public detail::guard_ptr> { + using base = detail::guard_ptr; + using Deleter = typename T::Deleter; + +public: + // Guard a marked ptr. + explicit guard_ptr(const MarkedPtr& p = MarkedPtr()) noexcept; + guard_ptr(const guard_ptr& p) noexcept; + guard_ptr(guard_ptr&& p) noexcept; + + guard_ptr& operator=(const guard_ptr& p) noexcept; + guard_ptr& operator=(guard_ptr&& p) noexcept; + + // Atomically take snapshot of p, and *if* it points to unreclaimed object, acquire shared ownership of it. + void acquire(const concurrent_ptr& p, std::memory_order order = std::memory_order_seq_cst) noexcept; + + // Like acquire, but quit early if a snapshot != expected. + bool acquire_if_equal(const concurrent_ptr& p, + const MarkedPtr& expected, + std::memory_order order = std::memory_order_seq_cst) noexcept; + + // Release ownership. Postcondition: get() == nullptr. + void reset() noexcept; + + // Reset. Deleter d will be applied some time after all owners release their ownership. + void reclaim(Deleter d = Deleter()) noexcept; +}; +} // namespace xenium::reclamation + +#define STAMP_IT_IMPL +#include +#undef STAMP_IT_IMPL + +#endif diff --git a/xenium/seqlock.hpp b/include/xenium/seqlock.hpp similarity index 100% rename from xenium/seqlock.hpp rename to include/xenium/seqlock.hpp diff --git a/xenium/utils.hpp b/include/xenium/utils.hpp similarity index 100% rename from xenium/utils.hpp rename to include/xenium/utils.hpp diff --git a/xenium/vyukov_bounded_queue.hpp b/include/xenium/vyukov_bounded_queue.hpp similarity index 100% rename from xenium/vyukov_bounded_queue.hpp rename to include/xenium/vyukov_bounded_queue.hpp diff --git a/xenium/vyukov_hash_map.hpp b/include/xenium/vyukov_hash_map.hpp similarity index 100% rename from xenium/vyukov_hash_map.hpp rename to include/xenium/vyukov_hash_map.hpp diff --git a/test/nikolaev_queue_test.cpp b/test/nikolaev_queue_test.cpp index 66e0284..5070cd5 100644 --- a/test/nikolaev_queue_test.cpp +++ b/test/nikolaev_queue_test.cpp @@ -7,7 +7,7 @@ #include #include "helpers.hpp" -#include "test/helpers.hpp" +// #include "test/helpers.hpp" #include