From c15b1c92ac3ea1b7a626a4eefffe188722fe9015 Mon Sep 17 00:00:00 2001 From: Harun Mustafa Date: Mon, 9 Dec 2024 12:42:55 +0100 Subject: [PATCH 01/17] inline sshash::perc --- include/info.impl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/info.impl b/include/info.impl index a62ad30..5ad5f3b 100644 --- a/include/info.impl +++ b/include/info.impl @@ -33,7 +33,7 @@ static double bits_per_kmer_formula(uint64_t k, /* kmer length */ return num_bits / n; } -double perc(uint64_t amount, uint64_t total) { return (amount * 100.0) / total; } +inline double perc(uint64_t amount, uint64_t total) { return (amount * 100.0) / total; } template void dictionary::print_space_breakdown() const { From c62d1ae0d2c194d36638b565d2dc488532d28613 Mon Sep 17 00:00:00 2001 From: Harun Mustafa Date: Mon, 9 Dec 2024 14:41:18 +0100 Subject: [PATCH 02/17] suppress uninitialized warning --- include/kmer.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/kmer.hpp b/include/kmer.hpp index 69affbe..c78ac7f 100644 --- a/include/kmer.hpp +++ b/include/kmer.hpp @@ -13,7 +13,7 @@ namespace sshash { template struct uint_kmer_t { using uint_t = Kmer; - Kmer kmer; + Kmer kmer = 0; uint_kmer_t() {} uint_kmer_t(uint64_t kmer) : kmer(kmer) {} From c86c661a1f82320e15f349d12ce849ee4641bcbb Mon Sep 17 00:00:00 2001 From: Harun Mustafa Date: Mon, 17 Mar 2025 19:31:10 +0100 Subject: [PATCH 03/17] Grant access to underlying dictionary (#11) --- include/dictionary.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/dictionary.hpp b/include/dictionary.hpp index 82a68cd..1ea3c44 100644 --- a/include/dictionary.hpp +++ b/include/dictionary.hpp @@ -137,6 +137,8 @@ struct dictionary { visit_impl(visitor, *this); } + const buckets& data() const { return m_buckets; } + private: template static void visit_impl(Visitor& visitor, T&& t) { From bedff077b7ef003fe59708733a35ef1a2746aef6 Mon Sep 17 00:00:00 2001 From: Harun Mustafa Date: Mon, 11 Aug 2025 16:51:02 +0200 Subject: [PATCH 04/17] Grant access to minimizers (#12) --- include/dictionary.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/dictionary.hpp b/include/dictionary.hpp index 1ea3c44..c599c61 100644 --- a/include/dictionary.hpp +++ b/include/dictionary.hpp @@ -138,6 +138,7 @@ struct dictionary { } const buckets& data() const { return m_buckets; } + const minimizers& get_minimizers() const { return m_minimizers; } private: template From 77f8e592fc5c9feed9bad6cea185a57af74bf7af Mon Sep 17 00:00:00 2001 From: Oleksandr Kulkov Date: Sat, 4 Oct 2025 18:37:01 +0200 Subject: [PATCH 05/17] include_directories -> target_include_directories --- CMakeLists.txt | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 617b38e..866c89c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,14 +54,6 @@ MESSAGE(STATUS "Installation prefix: ${CMAKE_INSTALL_PREFIX}") MESSAGE(STATUS "Compiling for processor: ${CMAKE_HOST_SYSTEM_PROCESSOR}") MESSAGE(STATUS "Compiling with flags:${CMAKE_CXX_FLAGS}") -include_directories(.) # all include paths relative to parent directory -include_directories(external/pthash/include) -include_directories(external/pthash/external/bits/include) -include_directories(external/pthash/external/fastmod) -include_directories(external/pthash/external/bits/external/essentials/include) -include_directories(external/pthash/external/xxHash) -include_directories(external/pthash/external/mm_file/include) - set(Z_LIB_SOURCES external/gz/zip_stream.cpp ) @@ -74,13 +66,27 @@ set(SSHASH_SOURCES src/statistics.cpp ) +set(SSHASH_INCLUDE_DIRS + external/pthash/include + external/pthash/external/bits/include + external/pthash/external/fastmod + external/pthash/external/bits/external/essentials/include + external/pthash/external/xxHash + external/pthash/external/mm_file/include + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/include +) + # Create a static lib add_library(sshash_static STATIC ${Z_LIB_SOURCES} ${SSHASH_SOURCES} ) +target_include_directories(sshash_static PUBLIC ${SSHASH_INCLUDE_DIRS}) + add_executable(sshash tools/sshash.cpp) +target_include_directories(sshash PUBLIC ${SSHASH_INCLUDE_DIRS}) target_link_libraries(sshash z ) From 238e817cecf9fa8e624af67d9d97e3e65cd22a17 Mon Sep 17 00:00:00 2001 From: Oleksandr Kulkov Date: Sat, 4 Oct 2025 19:07:03 +0200 Subject: [PATCH 06/17] Add canonicalize_basepair_reverse_map to Protein kmers --- include/kmer.hpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/include/kmer.hpp b/include/kmer.hpp index 002aa07..ea6b061 100644 --- a/include/kmer.hpp +++ b/include/kmer.hpp @@ -284,6 +284,19 @@ struct aa_uint_kmer_t : alpha_kmer_t { static bool is_valid(char c) { return ~char_to_aa[static_cast(c)]; } static uint64_t char_to_uint(char c) { return char_to_aa[static_cast(c)]; } + + // For proteins, there's no reverse complement, so map each character to itself + // This allows streaming_query to work with protein alphabets + static constexpr char canonicalize_basepair_reverse_map[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 0, 0, 0, 0, 0, + 0, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; }; // also supports bitpack<__uint128_t, 1>, std::bitset<256>, etc From d86f2eb814f7699a01827ef37932660ca29c6672 Mon Sep 17 00:00:00 2001 From: Oleksandr Kulkov Date: Sat, 4 Oct 2025 19:31:10 +0200 Subject: [PATCH 07/17] Fix Clang compilation and update pthash submodule - Add inline keyword to check_version_number static function in util.hpp to comply with Clang's stricter ODR checks on macOS arm64 - Update pthash submodule to cc4c9c9 with Clang fixes applied --- external/pthash | 2 +- include/util.hpp | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/external/pthash b/external/pthash index cc4c9c9..1022975 160000 --- a/external/pthash +++ b/external/pthash @@ -1 +1 @@ -Subproject commit cc4c9c9c7366f7ebe238b4b6493372e774608cbf +Subproject commit 1022975477bf889e5288bc616698319f6d6dddf4 diff --git a/include/util.hpp b/include/util.hpp index 3cb1247..7909dd9 100644 --- a/include/util.hpp +++ b/include/util.hpp @@ -170,11 +170,8 @@ struct build_configuration { namespace util { -static void check_version_number(essentials::version_number const& vnum) { - if (vnum.x != constants::current_version_number::x) { - throw std::runtime_error("MAJOR index version mismatch: SSHash index needs rebuilding"); - } -} +static inline void check_version_number(essentials::version_number const& vnum) { + if (vnum.major_version == 0 and vnum.minor_version == 0 and vnum.patch_version == 0) { static inline uint64_t get_seed_for_hash_function(build_configuration const& build_config) { static const uint64_t my_favourite_seed = 1234567890; From 0ea5acab3f28a34b9bc6d1d1b63bc715b069126d Mon Sep 17 00:00:00 2001 From: Oleksandr Kulkov Date: Sat, 4 Oct 2025 19:56:19 +0200 Subject: [PATCH 08/17] Fix Clang compilation --- .gitmodules | 2 +- include/util.hpp | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index a45b147..9f3579f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "external/pthash"] path = external/pthash - url = https://github.com/jermp/pthash.git + url = https://github.com/ratschlab/pthash.git diff --git a/include/util.hpp b/include/util.hpp index 7909dd9..e83fe8d 100644 --- a/include/util.hpp +++ b/include/util.hpp @@ -171,7 +171,10 @@ struct build_configuration { namespace util { static inline void check_version_number(essentials::version_number const& vnum) { - if (vnum.major_version == 0 and vnum.minor_version == 0 and vnum.patch_version == 0) { + if (vnum.x != constants::current_version_number::x) { + throw std::runtime_error("MAJOR index version mismatch: SSHash index needs rebuilding"); + } +} static inline uint64_t get_seed_for_hash_function(build_configuration const& build_config) { static const uint64_t my_favourite_seed = 1234567890; From d1abe66183d704ce6107cb8b5d26822ea81ae500 Mon Sep 17 00:00:00 2001 From: Oleksandr Kulkov Date: Sat, 4 Oct 2025 20:02:07 +0200 Subject: [PATCH 09/17] static -> static inline --- src/info.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/info.cpp b/src/info.cpp index 762cad6..89c5941 100644 --- a/src/info.cpp +++ b/src/info.cpp @@ -2,10 +2,10 @@ namespace sshash { -static double bits_per_kmer_formula(uint64_t k, /* kmer length */ - uint64_t m, /* minimizer length */ - uint64_t n, /* num. kmers */ - uint64_t M) /* num. strings in SPSS */ +static inline double bits_per_kmer_formula(uint64_t k, /* kmer length */ + uint64_t m, /* minimizer length */ + uint64_t n, /* num. kmers */ + uint64_t M) /* num. strings in SPSS */ { /* Caveats: From 452972f3ae6c07fa1a3a253b57a37eea61ab598b Mon Sep 17 00:00:00 2001 From: Oleksandr Kulkov Date: Sat, 4 Oct 2025 20:07:58 +0200 Subject: [PATCH 10/17] maybe_unused --- src/info.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/info.cpp b/src/info.cpp index 89c5941..27ccdf9 100644 --- a/src/info.cpp +++ b/src/info.cpp @@ -2,6 +2,7 @@ namespace sshash { +[[maybe_unused]] static inline double bits_per_kmer_formula(uint64_t k, /* kmer length */ uint64_t m, /* minimizer length */ uint64_t n, /* num. kmers */ From 424dea03035a75a85cf7f710fef2e4e0df13e829 Mon Sep 17 00:00:00 2001 From: Oleksandr Kulkov Date: Sat, 4 Oct 2025 20:27:56 +0200 Subject: [PATCH 11/17] Check if AVX2 is enabled instead of checking for x86_64 --- include/builder/parse_file.hpp | 2 +- include/builder/util.hpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/builder/parse_file.hpp b/include/builder/parse_file.hpp index 0a74583..1e71bb0 100644 --- a/include/builder/parse_file.hpp +++ b/include/builder/parse_file.hpp @@ -149,7 +149,7 @@ void parse_file(std::istream& is, parse_data& data, uint64_t i = 0; if constexpr (kmer_t::bits_per_char == 2) { -#if !defined(SSHASH_USE_TRADITIONAL_NUCLEOTIDE_ENCODING) and defined(__x86_64__) +#if !defined(SSHASH_USE_TRADITIONAL_NUCLEOTIDE_ENCODING) and defined(__AVX2__) /* process 32 bytes at a time */ for (; i + 32 <= n; i += 32) { diff --git a/include/builder/util.hpp b/include/builder/util.hpp index d40d40f..0b9b2cc 100644 --- a/include/builder/util.hpp +++ b/include/builder/util.hpp @@ -3,7 +3,7 @@ #include #include -#if defined(__x86_64__) +#if defined(__AVX2__) #include #include #endif @@ -30,7 +30,7 @@ struct parse_runtime_error : public std::runtime_error { } } -#if defined(__x86_64__) +#if defined(__AVX2__) /* This function takes 32 bytes and packs the two bits in positions 1 and 2 (from right) of each byte into From 841661d59c1c79b95d3f432a706cbf76bb69a1a9 Mon Sep 17 00:00:00 2001 From: Oleksandr Kulkov Date: Sat, 4 Oct 2025 23:48:47 +0200 Subject: [PATCH 12/17] Update pthash --- external/pthash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/pthash b/external/pthash index 1022975..6a3dbe5 160000 --- a/external/pthash +++ b/external/pthash @@ -1 +1 @@ -Subproject commit 1022975477bf889e5288bc616698319f6d6dddf4 +Subproject commit 6a3dbe54f77bf8c8efc8adeed42a161641588ecd From 7758aa8b7ba87687115090cb5b4688033a50e83a Mon Sep 17 00:00:00 2001 From: Oleksandr Kulkov Date: Sun, 5 Oct 2025 12:04:47 +0200 Subject: [PATCH 13/17] Update pthash --- external/pthash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/pthash b/external/pthash index 6a3dbe5..725846c 160000 --- a/external/pthash +++ b/external/pthash @@ -1 +1 @@ -Subproject commit 6a3dbe54f77bf8c8efc8adeed42a161641588ecd +Subproject commit 725846c18709efb4d2b55170c70ea174d73d62dc From a0859be0bdb6b53b381d71875badd9d566fbb192 Mon Sep 17 00:00:00 2001 From: Oleksandr Kulkov Date: Sun, 5 Oct 2025 14:15:37 +0200 Subject: [PATCH 14/17] revert pthash to upstreaam --- .gitmodules | 2 +- external/pthash | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 9f3579f..a45b147 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "external/pthash"] path = external/pthash - url = https://github.com/ratschlab/pthash.git + url = https://github.com/jermp/pthash.git diff --git a/external/pthash b/external/pthash index 725846c..cc4c9c9 160000 --- a/external/pthash +++ b/external/pthash @@ -1 +1 @@ -Subproject commit 725846c18709efb4d2b55170c70ea174d73d62dc +Subproject commit cc4c9c9c7366f7ebe238b4b6493372e774608cbf From 9399b62c631ca13e212364c1849fdba3fedc8823 Mon Sep 17 00:00:00 2001 From: Oleksandr Kulkov Date: Sun, 5 Oct 2025 14:38:34 +0200 Subject: [PATCH 15/17] Support any number of threads --- include/builder/parallel_sort.hpp | 4 +++- src/build.cpp | 3 --- tools/build.cpp | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/include/builder/parallel_sort.hpp b/include/builder/parallel_sort.hpp index 66668e0..a12848a 100644 --- a/include/builder/parallel_sort.hpp +++ b/include/builder/parallel_sort.hpp @@ -65,7 +65,6 @@ void parallel_sort(std::vector& data, const uint64_t num_threads, Compare com return; } - assert((num_threads & (num_threads - 1)) == 0); const uint64_t data_size = data.size(); const uint64_t chunk_size = (data_size + num_threads - 1) / num_threads; const uint64_t sequential_merge_threshold = data_size / uint64_t(std::log2(num_threads)); @@ -114,6 +113,9 @@ void parallel_sort(std::vector& data, const uint64_t num_threads, Compare com auto merged_begin = output_iterator; auto merged_end = merged_begin + output_size; next_ranges.push_back({merged_begin, merged_end}); + } else { + // Odd range out: carry it forward to the next iteration + next_ranges.push_back(ranges[i]); } } ranges.swap(next_ranges); diff --git a/src/build.cpp b/src/build.cpp index 759326b..15ab344 100644 --- a/src/build.cpp +++ b/src/build.cpp @@ -28,9 +28,6 @@ void dictionary::build(std::string const& filename, if (build_config.l >= constants::max_l) { throw std::runtime_error("l must be < " + std::to_string(constants::max_l)); } - if ((build_config.num_threads & (build_config.num_threads - 1)) != 0) { - throw std::runtime_error("number of threads must be a power of 2"); - } m_k = build_config.k; m_m = build_config.m; diff --git a/tools/build.cpp b/tools/build.cpp index cb38f14..9c9d738 100644 --- a/tools/build.cpp +++ b/tools/build.cpp @@ -21,7 +21,7 @@ int build(int argc, char** argv) { parser.add("seed", "Seed for construction (default is " + std::to_string(constants::seed) + ").", "-s", false); - parser.add("t", "Number of threads (default is 1). Must be a power of 2.", "-t", false); + parser.add("t", "Number of threads (default is 1).", "-t", false); parser.add("l", "A (integer) constant that controls the space/time trade-off of the dictionary. " "A reasonable values lies in [2.." + From 4a80d203289a103356f9879b8246d6f3182d6d80 Mon Sep 17 00:00:00 2001 From: Oleksandr Kulkov Date: Sun, 5 Oct 2025 15:32:58 +0200 Subject: [PATCH 16/17] Fix parallel_sort.hpp --- include/builder/parallel_sort.hpp | 36 +++++++++++++++---------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/include/builder/parallel_sort.hpp b/include/builder/parallel_sort.hpp index a12848a..c1f531e 100644 --- a/include/builder/parallel_sort.hpp +++ b/include/builder/parallel_sort.hpp @@ -55,17 +55,18 @@ void parallel_merge(Iterator A_begin, Iterator A_end, // working memory, which must be of same size as data. */ template -void parallel_sort(std::vector& data, const uint64_t num_threads, Compare comp) // +void parallel_sort(std::vector& data, uint64_t num_threads, Compare comp) // { std::vector temp_data; temp_data.resize(data.size()); + const uint64_t data_size = data.size(); + num_threads = std::min(num_threads, data_size); if (num_threads <= 1) { std::sort(data.begin(), data.end(), comp); return; } - const uint64_t data_size = data.size(); const uint64_t chunk_size = (data_size + num_threads - 1) / num_threads; const uint64_t sequential_merge_threshold = data_size / uint64_t(std::log2(num_threads)); assert(sequential_merge_threshold > 0); @@ -93,30 +94,29 @@ void parallel_sort(std::vector& data, const uint64_t num_threads, Compare com while (ranges.size() != 1) // { next_ranges.clear(); - for (uint64_t i = 0; i != ranges.size(); i += 2) { + for (uint64_t i = 0; i < ranges.size(); i += 2) { + auto [begin1, end1] = ranges[i]; + + auto input = data.begin(); + auto output = temp_data.begin(); + if (swap) std::swap(input, output); + uint64_t offset = std::distance(input, begin1); + auto output_iterator = output + offset; + assert(offset <= data_size); + uint64_t output_size = end1 - begin1; if (i + 1 < ranges.size()) { - auto [begin1, end1] = ranges[i]; auto [begin2, end2] = ranges[i + 1]; - uint64_t output_size = (end1 - begin1) + (end2 - begin2); - - auto input = data.begin(); - auto output = temp_data.begin(); - if (swap) std::swap(input, output); - uint64_t offset = std::distance(input, begin1); - auto output_iterator = output + offset; - assert(offset <= data_size); + output_size += end2 - begin2; parallel_merge(begin1, end1, begin2, end2, output_iterator, comp, sequential_merge_threshold); assert(std::is_sorted(output_iterator, output_iterator + output_size, comp)); - - auto merged_begin = output_iterator; - auto merged_end = merged_begin + output_size; - next_ranges.push_back({merged_begin, merged_end}); } else { - // Odd range out: carry it forward to the next iteration - next_ranges.push_back(ranges[i]); + std::move(begin1, end1, output_iterator); } + auto merged_begin = output_iterator; + auto merged_end = merged_begin + output_size; + next_ranges.push_back({merged_begin, merged_end}); } ranges.swap(next_ranges); swap = !swap; From d87e11fa6f30f59d1ebc1b8ea383ae5ea5df080e Mon Sep 17 00:00:00 2001 From: Oleksandr Kulkov Date: Sun, 5 Oct 2025 19:03:10 +0200 Subject: [PATCH 17/17] Multithreading fixes --- include/builder/build_sparse_index.hpp | 15 ++++++++------- include/builder/parallel_sort.hpp | 9 +++++---- include/builder/parse_file.hpp | 4 ++-- src/build.cpp | 6 +++--- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/include/builder/build_sparse_index.hpp b/include/builder/build_sparse_index.hpp index 5db9084..fe9ccdb 100644 --- a/include/builder/build_sparse_index.hpp +++ b/include/builder/build_sparse_index.hpp @@ -82,9 +82,9 @@ buckets_statistics build_sparse_index(parse_data& data, buckets& } } offsets.push_back(num_super_kmers); - assert(offsets.size() == num_threads + 1); - std::vector threads_buckets_stats(num_threads); + std::vector threads_buckets_stats; + threads_buckets_stats.reserve(num_threads); auto exe = [&](const uint64_t thread_id) { assert(thread_id + 1 < offsets.size()); @@ -115,11 +115,12 @@ buckets_statistics build_sparse_index(parse_data& data, buckets& } }; - std::vector threads(num_threads); - for (uint64_t thread_id = 0; thread_id != num_threads; ++thread_id) { - threads_buckets_stats[thread_id] = - buckets_statistics(num_buckets, num_kmers, num_minimizer_positions); - threads[thread_id] = std::thread(exe, thread_id); + std::vector threads; + threads.reserve(num_threads); + assert(offsets.size() <= num_threads + 1); + for (uint64_t thread_id = 0; thread_id + 1 < size(offsets); ++thread_id) { + threads_buckets_stats.emplace_back(num_buckets, num_kmers, num_minimizer_positions); + threads.emplace_back(exe, thread_id); } for (auto& t : threads) { if (t.joinable()) t.join(); diff --git a/include/builder/parallel_sort.hpp b/include/builder/parallel_sort.hpp index c1f531e..9b6c63c 100644 --- a/include/builder/parallel_sort.hpp +++ b/include/builder/parallel_sort.hpp @@ -75,12 +75,13 @@ void parallel_sort(std::vector& data, uint64_t num_threads, Compare comp) // threads.reserve(num_threads); using iterator_t = typename std::vector::iterator; - std::vector> ranges(num_threads); + std::vector> ranges; + ranges.reserve(num_threads); - for (uint64_t i = 0; i != num_threads; ++i) { + for (uint64_t i = 0; i * chunk_size < data_size; ++i) { uint64_t begin = i * chunk_size; - uint64_t end = (i == num_threads - 1) ? data_size : begin + chunk_size; - ranges[i] = {data.begin() + begin, data.begin() + end}; + uint64_t end = std::min(data_size, begin + chunk_size); + ranges.emplace_back(data.begin() + begin, data.begin() + end); threads.emplace_back( [&, begin, end]() { std::sort(data.begin() + begin, data.begin() + end, comp); }); } diff --git a/include/builder/parse_file.hpp b/include/builder/parse_file.hpp index 1e71bb0..f269788 100644 --- a/include/builder/parse_file.hpp +++ b/include/builder/parse_file.hpp @@ -214,7 +214,7 @@ void parse_file(std::istream& is, parse_data& data, std::vector threads; threads.reserve(num_threads); - for (uint64_t t = 0; t != num_threads; ++t) // + for (uint64_t t = 0; t * num_sequences_per_thread < num_sequences; ++t) // { threads.emplace_back([&, t] { std::vector buffer; @@ -249,7 +249,7 @@ void parse_file(std::istream& is, parse_data& data, minimizer_iterator minimizer_it(k, m, hasher); minimizer_iterator_rc minimizer_it_rc(k, m, hasher); - for (uint64_t i = index_begin; i != index_end; ++i) // + for (uint64_t i = index_begin; i < index_end; ++i) // { const uint64_t begin = data.pieces[i]; const uint64_t end = data.pieces[i + 1]; diff --git a/src/build.cpp b/src/build.cpp index 15ab344..117f2d7 100644 --- a/src/build.cpp +++ b/src/build.cpp @@ -123,11 +123,11 @@ void dictionary::build(std::string const& filename, input.read(reinterpret_cast(buffer.data()), buffer.size() * sizeof(minimizer_tuple)); const uint64_t chunk_size = (n + num_threads - 1) / num_threads; - for (uint64_t t = 0; t != num_threads; ++t) { + for (uint64_t t = 0; t * chunk_size < n; ++t) { uint64_t begin = t * chunk_size; - uint64_t end = (t == num_threads - 1) ? n : begin + chunk_size; + uint64_t end = std::min(n, begin + chunk_size); threads.emplace_back([begin, end, &buffer, &f]() { - for (uint64_t i = begin; i != end; ++i) { + for (uint64_t i = begin; i < end; ++i) { buffer[i].minimizer = f.lookup(buffer[i].minimizer); } });