From e0485c31c7394ba6e378ca9aca0701be295908df Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Mon, 6 Jan 2025 18:12:03 +0100 Subject: [PATCH] InvertedIndexFindByKeyQuery --- cmake/compiler_settings.cmake | 10 +- silkworm/db/datastore/common/owning_view.hpp | 112 ++++++++++ silkworm/db/datastore/common/timestamp.hpp | 3 + silkworm/db/datastore/concat_view.hpp | 165 +++++++++++++++ silkworm/db/datastore/concat_view_test.cpp | 61 ++++++ .../inverted_index_range_by_key_query.hpp | 71 +++++++ .../db/datastore/kvdb/cursor_iterator.cpp | 47 +++++ .../db/datastore/kvdb/cursor_iterator.hpp | 199 ++++++++++++++++++ silkworm/db/datastore/kvdb/database.cpp | 11 + silkworm/db/datastore/kvdb/database.hpp | 2 + .../datastore/kvdb/inverted_index_queries.hpp | 1 + .../inverted_index_range_by_key_query.hpp | 94 +++++++++ ...inverted_index_range_by_key_query_test.cpp | 108 ++++++++++ .../common/util/iterator/map_values_view.hpp | 2 +- .../db/datastore/snapshots/inverted_index.hpp | 2 +- .../snapshots/inverted_index_queries.hpp | 19 ++ .../inverted_index_range_by_key_query.hpp | 108 ++++++++++ .../snapshots/snapshot_repository.cpp | 32 ++- .../snapshots/snapshot_repository.hpp | 2 + .../snapshot_repository_ro_access.hpp | 8 + silkworm/db/kv/api/local_transaction.cpp | 31 ++- 21 files changed, 1078 insertions(+), 10 deletions(-) create mode 100644 silkworm/db/datastore/common/owning_view.hpp create mode 100644 silkworm/db/datastore/concat_view.hpp create mode 100644 silkworm/db/datastore/concat_view_test.cpp create mode 100644 silkworm/db/datastore/inverted_index_range_by_key_query.hpp create mode 100644 silkworm/db/datastore/kvdb/cursor_iterator.cpp create mode 100644 silkworm/db/datastore/kvdb/cursor_iterator.hpp create mode 100644 silkworm/db/datastore/kvdb/inverted_index_range_by_key_query.hpp create mode 100644 silkworm/db/datastore/kvdb/inverted_index_range_by_key_query_test.cpp create mode 100644 silkworm/db/datastore/snapshots/inverted_index_queries.hpp create mode 100644 silkworm/db/datastore/snapshots/inverted_index_range_by_key_query.hpp diff --git a/cmake/compiler_settings.cmake b/cmake/compiler_settings.cmake index 6d865906e0..9cb2957466 100644 --- a/cmake/compiler_settings.cmake +++ b/cmake/compiler_settings.cmake @@ -79,11 +79,15 @@ elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES ".*Clang$") add_link_options(-fprofile-instr-generate -fcoverage-mapping) endif() - # coroutines support + # configure libc++ if(NOT SILKWORM_WASM_API) add_compile_options($<$:-stdlib=libc++>) - if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 16) - add_compile_definitions($<$:_LIBCPP_ENABLE_EXPERIMENTAL>) + # std::views::join is experimental on clang < 18 and Apple clang < 16 + if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 18) + add_compile_options(-fexperimental-library) + endif() + if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 16) + add_compile_options(-fexperimental-library) endif() link_libraries(c++) link_libraries(c++abi) diff --git a/silkworm/db/datastore/common/owning_view.hpp b/silkworm/db/datastore/common/owning_view.hpp new file mode 100644 index 0000000000..94c347bee6 --- /dev/null +++ b/silkworm/db/datastore/common/owning_view.hpp @@ -0,0 +1,112 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include + +// std::ranges::owning_view is not present on GCC < 12.1 +// see P2415R2 at https://gcc.gnu.org/onlinedocs/libstdc++/manual/status.html#status.iso.2020 +#if __GNUC__ < 12 && !defined(__clang__) +#else +#define SILKWORM_HAS_BUILTIN_OWNING_VIEW +#endif + +#ifdef SILKWORM_HAS_BUILTIN_OWNING_VIEW +namespace silkworm::ranges::builtin { + +template +using OwningView = std::ranges::owning_view; + +} // namespace silkworm::ranges::builtin +#endif + +namespace silkworm::ranges::fallback { + +// https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2021/p2415r2.html +template + requires std::movable +class OwningView : public std::ranges::view_interface> { + public: + OwningView() + requires std::default_initializable + = default; + + explicit constexpr OwningView(TRange&& range) : range_{std::move(range)} {} + + OwningView(OwningView&&) = default; + OwningView& operator=(OwningView&&) = default; + + constexpr TRange& base() & noexcept { return range_; } + constexpr const TRange& base() const& noexcept { return range_; } + constexpr TRange&& base() && noexcept { return std::move(range_); } + constexpr const TRange&& base() const&& noexcept { return std::move(range_); } + + constexpr std::ranges::iterator_t begin() { return std::ranges::begin(range_); } + constexpr std::ranges::sentinel_t end() { return std::ranges::end(range_); } + + constexpr auto begin() const + requires std::ranges::range + { return std::ranges::begin(range_); } + + constexpr auto end() const + requires std::ranges::range + { return std::ranges::end(range_); } + + constexpr bool empty() + requires requires { std::ranges::empty(std::declval()); } + { return std::ranges::empty(range_); } + + constexpr bool empty() const + requires requires { std::ranges::empty(std::declval()); } + { return std::ranges::empty(range_); } + + constexpr auto size() + requires std::ranges::sized_range + { return std::ranges::size(range_); } + + constexpr auto size() const + requires std::ranges::sized_range + { return std::ranges::size(range_); } + + constexpr auto data() + requires std::ranges::contiguous_range + { return std::ranges::data(range_); } + + constexpr auto data() const + requires std::ranges::contiguous_range + { return std::ranges::data(range_); } + + private: + TRange range_; +}; + +} // namespace silkworm::ranges::fallback + +namespace silkworm::ranges { + +#ifdef SILKWORM_HAS_BUILTIN_OWNING_VIEW +using silkworm::ranges::builtin::OwningView; +#else +using silkworm::ranges::fallback::OwningView; +#endif + +template +auto owning_view(TRange&& range) { + return OwningView{std::forward(range)}; +} + +} // namespace silkworm::ranges diff --git a/silkworm/db/datastore/common/timestamp.hpp b/silkworm/db/datastore/common/timestamp.hpp index 7268e0bc53..abf7d498d4 100644 --- a/silkworm/db/datastore/common/timestamp.hpp +++ b/silkworm/db/datastore/common/timestamp.hpp @@ -29,6 +29,9 @@ struct TimestampRange { TimestampRange(Timestamp start1, Timestamp end1) : start(start1), end(end1) {} friend bool operator==(const TimestampRange&, const TimestampRange&) = default; bool contains(Timestamp value) const { return (start <= value) && (value < end); } + auto contains_predicate() { + return [range = *this](Timestamp t) { return range.contains(t); }; + }; bool contains_range(TimestampRange range) const { return (start <= range.start) && (range.end <= end); } Timestamp size() const { return end - start; } std::string to_string() const { return std::string("[") + std::to_string(start) + ", " + std::to_string(end) + ")"; } diff --git a/silkworm/db/datastore/concat_view.hpp b/silkworm/db/datastore/concat_view.hpp new file mode 100644 index 0000000000..3670575468 --- /dev/null +++ b/silkworm/db/datastore/concat_view.hpp @@ -0,0 +1,165 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include +#include +#include + +#include + +// std::views::concat is present on C++26 +#if __cplusplus >= 202601L +#define SILKWORM_HAS_BUILTIN_CONCAT_VIEW +#endif + +#ifdef SILKWORM_HAS_BUILTIN_CONCAT_VIEW +namespace silkworm::views::concat_view::builtin { + +template +using ConcatView = std::ranges::concat_view; + +} // namespace silkworm::views::concat_view::builtin +#endif + +namespace silkworm::views::concat_view::fallback { + +template +class ConcatView : public std::ranges::view_interface> { + public: + class Iterator { + public: + using Range1Iterator = std::ranges::iterator_t; + using Range1Sentinel = std::ranges::sentinel_t; + using Range1ReferenceType = std::iter_reference_t; + using Range2Iterator = std::ranges::iterator_t; + using Range2Sentinel = std::ranges::sentinel_t; + using Range2ReferenceType = std::iter_reference_t; + using DereferenceType = std::conditional_t, Range1ReferenceType, Range2ReferenceType>; + + using value_type = std::iter_value_t; + using iterator_category [[maybe_unused]] = std::input_iterator_tag; + using difference_type = std::iter_difference_t; + using reference = DereferenceType; + using pointer = std::remove_reference_t*; + + Iterator() = default; + Iterator(Range1* range1, Range2* range2) + : range1_{range1}, + range2_{range2} { + it1_ = std::ranges::begin(*range1_); + sentinel1_ = std::ranges::end(*range1_); + if (*it1_ == *sentinel1_) { + it1_ = std::nullopt; + sentinel1_ = std::nullopt; + it2_ = std::ranges::begin(*range2_); + sentinel2_ = std::ranges::end(*range2_); + if (*it2_ == *sentinel2_) { + it2_ = std::nullopt; + sentinel2_ = std::nullopt; + } + } + } + + reference operator*() const { + if (it1_) return **it1_; + if (it2_) return **it2_; + SILKWORM_ASSERT(false); + std::abort(); + } + + Iterator operator++(int) { return std::exchange(*this, ++Iterator{*this}); } + Iterator& operator++() { + if (it1_) { + ++(*it1_); + if (*it1_ == *sentinel1_) { + it1_ = std::nullopt; + sentinel1_ = std::nullopt; + it2_ = std::ranges::begin(*range2_); + sentinel2_ = std::ranges::end(*range2_); + if (*it2_ == *sentinel2_) { + it2_ = std::nullopt; + sentinel2_ = std::nullopt; + } + } + } else if (it2_) { + ++(*it2_); +#if __GNUC__ < 12 && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif + if (*it2_ == *sentinel2_) { +#if __GNUC__ < 12 && !defined(__clang__) +#pragma GCC diagnostic pop +#endif + it2_ = std::nullopt; + sentinel2_ = std::nullopt; + } + } + return *this; + } + + friend bool operator==(const Iterator& it, const std::default_sentinel_t&) { + return !it.it1_ && !it.it2_; + } + friend bool operator!=(const Iterator& it, const std::default_sentinel_t&) { + return it.it1_ || it.it2_; + } + + private: + Range1* range1_{nullptr}; + Range2* range2_{nullptr}; + std::optional it1_; + std::optional it2_; + std::optional sentinel1_; + std::optional sentinel2_; + }; + + static_assert(std::input_iterator); + + ConcatView(Range1 range1, Range2 range2) + : range1_{std::move(range1)}, + range2_{std::move(range2)} {} + ConcatView() = default; + + ConcatView(ConcatView&&) = default; + ConcatView& operator=(ConcatView&&) noexcept = default; + + Iterator begin() { return Iterator{&range1_, &range2_}; } + std::default_sentinel_t end() const { return std::default_sentinel; } + + private: + Range1 range1_; + Range2 range2_; +}; + +} // namespace silkworm::views::concat_view::fallback + +namespace silkworm::views { + +#ifdef SILKWORM_HAS_BUILTIN_CONCAT_VIEW +using silkworm::views::concat_view::builtin::ConcatView; +#else +using silkworm::views::concat_view::fallback::ConcatView; +#endif + +template +auto concat(Range1&& v1, Range2&& v2) { + return ConcatView{std::forward(v1), std::forward(v2)}; +} + +} // namespace silkworm::views diff --git a/silkworm/db/datastore/concat_view_test.cpp b/silkworm/db/datastore/concat_view_test.cpp new file mode 100644 index 0000000000..1ed6672e9f --- /dev/null +++ b/silkworm/db/datastore/concat_view_test.cpp @@ -0,0 +1,61 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "concat_view.hpp" + +#include +#include +#include + +#include + +#include "common/owning_view.hpp" + +namespace silkworm::views::concat_view { + +static_assert(std::ranges::input_range, std::vector>>); + +template >> +std::vector vector_from_range(Range range) { + std::vector results; + std::ranges::copy(range, std::back_inserter(results)); + return results; +} + +TEST_CASE("ConcatView") { + CHECK(vector_from_range(concat( + silkworm::ranges::owning_view(std::vector{1, 2, 3}), + silkworm::ranges::owning_view(std::vector{4, 5, 6}))) == std::vector{1, 2, 3, 4, 5, 6}); + + auto even = [](int x) { return x % 2 == 0; }; + auto odd = [](int x) { return x % 2 == 1; }; + CHECK(vector_from_range(concat( + silkworm::ranges::owning_view(std::vector{1, 2, 3}) | std::views::filter(even), + silkworm::ranges::owning_view(std::vector{4, 5, 6}) | std::views::filter(odd))) == std::vector{2, 5}); + CHECK(vector_from_range(concat( + silkworm::ranges::owning_view(std::vector{1, 2, 3}) | std::views::filter(odd), + silkworm::ranges::owning_view(std::vector{4, 5, 6}) | std::views::filter(even))) == std::vector{1, 3, 4, 6}); + + CHECK(vector_from_range(concat(std::ranges::empty_view{}, std::ranges::empty_view{})).empty()); + CHECK(vector_from_range(concat(silkworm::ranges::owning_view(std::vector{1, 2, 3}), std::ranges::empty_view{})) == std::vector{1, 2, 3}); + CHECK(vector_from_range(concat(std::ranges::empty_view{}, silkworm::ranges::owning_view(std::vector{4, 5, 6}))) == std::vector{4, 5, 6}); + + CHECK(vector_from_range(concat( + silkworm::ranges::owning_view(std::vector{1, 2, 3}) | std::views::transform([](int v) { return std::vector{v, v, v}; }) | std::views::join, + silkworm::ranges::owning_view(std::vector{4, 4, 4}))) == std::vector{1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4}); +} + +} // namespace silkworm::views::concat_view diff --git a/silkworm/db/datastore/inverted_index_range_by_key_query.hpp b/silkworm/db/datastore/inverted_index_range_by_key_query.hpp new file mode 100644 index 0000000000..312bfe172f --- /dev/null +++ b/silkworm/db/datastore/inverted_index_range_by_key_query.hpp @@ -0,0 +1,71 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include "concat_view.hpp" +#include "kvdb/database.hpp" +#include "kvdb/inverted_index_range_by_key_query.hpp" +#include "snapshots/inverted_index_range_by_key_query.hpp" + +namespace silkworm::datastore { + +template +struct InvertedIndexRangeByKeyQuery { + InvertedIndexRangeByKeyQuery( + datastore::EntityName inverted_index_name, + kvdb::InvertedIndex kvdb_inverted_index, + kvdb::ROTxn& tx, + const snapshots::SnapshotRepositoryROAccess& repository) + : query1_{tx, kvdb_inverted_index}, + query2_{repository, inverted_index_name} {} + + InvertedIndexRangeByKeyQuery( + datastore::EntityName inverted_index_name, + kvdb::DatabaseRef database, + kvdb::ROTxn& tx, + const snapshots::SnapshotRepositoryROAccess& repository) + : InvertedIndexRangeByKeyQuery{ + inverted_index_name, + database.inverted_index(inverted_index_name), + tx, + repository, + } {} + + using TKey1 = decltype(TKeyEncoder1::value); + using TKey2 = decltype(TKeyEncoder2::value); + static_assert(std::same_as); + using TKey = TKey1; + + template + auto exec(TKey key, TimestampRange ts_range) { + if constexpr (ascending) { + return silkworm::views::concat( + query2_.template exec(key, ts_range), + query1_.exec(key, ts_range, ascending)); + } else { + return silkworm::views::concat( + query1_.exec(key, ts_range, ascending), + query2_.template exec(key, ts_range)); + } + } + + private: + kvdb::InvertedIndexRangeByKeyQuery query1_; + snapshots::InvertedIndexRangeByKeyQuery query2_; +}; + +} // namespace silkworm::datastore diff --git a/silkworm/db/datastore/kvdb/cursor_iterator.cpp b/silkworm/db/datastore/kvdb/cursor_iterator.cpp new file mode 100644 index 0000000000..b7bd9a4f32 --- /dev/null +++ b/silkworm/db/datastore/kvdb/cursor_iterator.cpp @@ -0,0 +1,47 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "cursor_iterator.hpp" + +#include "raw_codec.hpp" + +namespace silkworm::datastore::kvdb { + +void CursorIterator::decode(const CursorResult& result) { + if (result) { + if (decoders_.first) { + decoders_.first->decode(result.key); + } + if (decoders_.second) { + decoders_.second->decode(result.value); + } + } else { + decoders_.first.reset(); + decoders_.second.reset(); + } +} + +bool operator==(const CursorIterator& lhs, const CursorIterator& rhs) { + return (lhs.decoders_ == rhs.decoders_) && + ((!lhs.decoders_.first && !lhs.decoders_.second) || (lhs.cursor_ == rhs.cursor_)); +} + +static_assert(std::input_iterator); +static_assert(std::input_iterator, RawDecoder>>); +static_assert(std::input_iterator>>); +static_assert(std::input_iterator>>); + +} // namespace silkworm::datastore::kvdb diff --git a/silkworm/db/datastore/kvdb/cursor_iterator.hpp b/silkworm/db/datastore/kvdb/cursor_iterator.hpp new file mode 100644 index 0000000000..e0ceff2cd4 --- /dev/null +++ b/silkworm/db/datastore/kvdb/cursor_iterator.hpp @@ -0,0 +1,199 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include +#include +#include + +#include "codec.hpp" +#include "mdbx.hpp" + +namespace silkworm::datastore::kvdb { + +class CursorIterator { + public: + using value_type = std::pair, std::shared_ptr>; + using iterator_category [[maybe_unused]] = std::input_iterator_tag; + using difference_type = std::ptrdiff_t; + using pointer = value_type*; + using reference = value_type&; + + CursorIterator() = default; + + CursorIterator( + std::shared_ptr cursor, + MoveOperation move_op, + std::shared_ptr key_decoder, + std::shared_ptr value_decoder) + : cursor_{std::move(cursor)}, + move_op_{move_op}, + decoders_{std::move(key_decoder), std::move(value_decoder)} { + decode(cursor_->current(false)); + } + + value_type operator*() const { return decoders_; } + const value_type* operator->() const { return &decoders_; } + + CursorIterator operator++(int) { return std::exchange(*this, ++CursorIterator{*this}); } + CursorIterator& operator++() { + decode(cursor_->move(move_op_, false)); + return *this; + } + + friend bool operator!=(const CursorIterator& lhs, const CursorIterator& rhs) = default; + friend bool operator==(const CursorIterator& lhs, const CursorIterator& rhs); + + private: + void decode(const CursorResult& result); + std::shared_ptr cursor_; + MoveOperation move_op_; + value_type decoders_; +}; + +template +class CursorKVIterator { + public: + using value_type_owned = std::pair; + using value_type = std::pair; + using iterator_category [[maybe_unused]] = std::input_iterator_tag; + using difference_type = std::ptrdiff_t; + using pointer = value_type*; + using reference = value_type&; + + CursorKVIterator() = default; + + explicit CursorKVIterator(CursorIterator it) + : it_{std::move(it)} {} + + static CursorKVIterator make(std::unique_ptr cursor, MoveOperation move_op) { + return CursorKVIterator{CursorIterator{std::move(cursor), move_op, std::make_shared(), std::make_shared()}}; + } + + value_type operator*() const { return value(); } + + value_type_owned move_value() const { + value_type value = this->value(); + return {std::move(value.first), std::move(value.second)}; + } + + CursorKVIterator operator++(int) { return std::exchange(*this, ++CursorKVIterator{*this}); } + CursorKVIterator& operator++() { + ++it_; + return *this; + } + + friend bool operator!=(const CursorKVIterator& lhs, const CursorKVIterator& rhs) = default; + friend bool operator==(const CursorKVIterator& lhs, const CursorKVIterator& rhs) = default; + + private: + value_type value() const { + Decoder& base_key_decoder = *(it_->first); + Decoder& base_value_decoder = *(it_->second); + // dynamic_cast is safe if TKeyDecoder was used when creating the CursorIterator + auto& key_decoder = dynamic_cast(base_key_decoder); + // dynamic_cast is safe if TValueDecoder was used when creating the CursorIterator + auto& key_value_decoder = dynamic_cast(base_value_decoder); + return {key_decoder.value, key_value_decoder.value}; + } + + CursorIterator it_; +}; + +template +class CursorKeysIterator { + public: + using value_type = decltype(TKeyDecoder::value); + using iterator_category [[maybe_unused]] = std::input_iterator_tag; + using difference_type = std::ptrdiff_t; + using pointer = value_type*; + using reference = value_type&; + + CursorKeysIterator() = default; + + explicit CursorKeysIterator(CursorIterator it) + : it_{std::move(it)} {} + + static CursorKeysIterator make(std::unique_ptr cursor, MoveOperation move_op) { + return CursorKeysIterator{CursorIterator{std::move(cursor), move_op, std::make_shared(), {}}}; + } + + reference operator*() const { return value(); } + pointer operator->() const { return &value(); } + + CursorKeysIterator operator++(int) { return std::exchange(*this, ++CursorKeysIterator{*this}); } + CursorKeysIterator& operator++() { + ++it_; + return *this; + } + + friend bool operator!=(const CursorKeysIterator& lhs, const CursorKeysIterator& rhs) = default; + friend bool operator==(const CursorKeysIterator& lhs, const CursorKeysIterator& rhs) = default; + + private: + value_type& value() const { + Decoder& base_key_decoder = *(it_->first); + // dynamic_cast is safe if TKeyDecoder was used when creating the CursorIterator + auto& key_decoder = dynamic_cast(base_key_decoder); + return key_decoder.value; + } + + CursorIterator it_; +}; + +template +class CursorValuesIterator { + public: + using value_type = decltype(TValueDecoder::value); + using iterator_category [[maybe_unused]] = std::input_iterator_tag; + using difference_type = std::ptrdiff_t; + using pointer = value_type*; + using reference = value_type&; + + CursorValuesIterator() = default; + + explicit CursorValuesIterator(CursorIterator it) + : it_{std::move(it)} {} + + static CursorValuesIterator make(std::unique_ptr cursor, MoveOperation move_op) { + return CursorValuesIterator{CursorIterator{std::move(cursor), move_op, {}, std::make_shared()}}; + } + + reference operator*() const { return value(); } + pointer operator->() const { return &value(); } + + CursorValuesIterator operator++(int) { return std::exchange(*this, ++CursorValuesIterator{*this}); } + CursorValuesIterator& operator++() { + ++it_; + return *this; + } + + friend bool operator!=(const CursorValuesIterator& lhs, const CursorValuesIterator& rhs) = default; + friend bool operator==(const CursorValuesIterator& lhs, const CursorValuesIterator& rhs) = default; + + private: + value_type& value() const { + Decoder& base_value_decoder = *(it_->second); + // dynamic_cast is safe if TValueDecoder was used when creating the CursorIterator + auto& value_decoder = dynamic_cast(base_value_decoder); + return value_decoder.value; + } + + CursorIterator it_; +}; + +} // namespace silkworm::datastore::kvdb diff --git a/silkworm/db/datastore/kvdb/database.cpp b/silkworm/db/datastore/kvdb/database.cpp index e55945f6c7..9fd696500f 100644 --- a/silkworm/db/datastore/kvdb/database.cpp +++ b/silkworm/db/datastore/kvdb/database.cpp @@ -44,6 +44,17 @@ DatabaseRef::EntitiesMap make_entities( return results; } +void Database::create_tables() { + RWTxnManaged tx = access_rw().start_rw_tx(); + for (auto& entity : entities_) { + for (auto& entry : entity.second) { + MapConfig& map_config = entry.second; + tx->create_map(map_config.name, map_config.key_mode, map_config.value_mode); + } + } + tx.commit_and_stop(); +} + Domain DatabaseRef::domain(datastore::EntityName name) const { auto& entity = entities_.at(name); auto& domain_def = dynamic_cast(*schema_.entities().at(name)); diff --git a/silkworm/db/datastore/kvdb/database.hpp b/silkworm/db/datastore/kvdb/database.hpp index de8c42354c..36c42af19c 100644 --- a/silkworm/db/datastore/kvdb/database.hpp +++ b/silkworm/db/datastore/kvdb/database.hpp @@ -75,6 +75,8 @@ class Database { DatabaseRef ref() const { return {env_, schema_, entities_}; } // NOLINT(cppcoreguidelines-slicing) + void create_tables(); + private: mdbx::env_managed env_; Schema::DatabaseDef schema_; diff --git a/silkworm/db/datastore/kvdb/inverted_index_queries.hpp b/silkworm/db/datastore/kvdb/inverted_index_queries.hpp index 3fd4f3c56d..d93fbd116c 100644 --- a/silkworm/db/datastore/kvdb/inverted_index_queries.hpp +++ b/silkworm/db/datastore/kvdb/inverted_index_queries.hpp @@ -17,3 +17,4 @@ #pragma once #include "inverted_index_put_query.hpp" +#include "inverted_index_range_by_key_query.hpp" diff --git a/silkworm/db/datastore/kvdb/inverted_index_range_by_key_query.hpp b/silkworm/db/datastore/kvdb/inverted_index_range_by_key_query.hpp new file mode 100644 index 0000000000..922c863017 --- /dev/null +++ b/silkworm/db/datastore/kvdb/inverted_index_range_by_key_query.hpp @@ -0,0 +1,94 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include +#include +#include + +#include "../common/timestamp.hpp" +#include "codec.hpp" +#include "cursor_iterator.hpp" +#include "inverted_index.hpp" +#include "mdbx.hpp" +#include "timestamp_codec.hpp" + +namespace silkworm::datastore::kvdb { + +template +struct InvertedIndexRangeByKeyQuery { + ROTxn& tx; + InvertedIndex entity; + + using TKey = decltype(TKeyEncoder::value); + + //! A range of timestamps + using Timestamps = std::ranges::filter_view< + std::ranges::subrange>, + decltype(std::declval().contains_predicate())>; + + CursorValuesIterator begin(TKey key, TimestampRange ts_range, bool ascending) { + auto cursor = tx.ro_cursor_dup_sort(entity.index_table); + + TKeyEncoder key_encoder; + key_encoder.value = std::move(key); + Slice key_data = key_encoder.encode(); + + TimestampEncoder ts_encoder; + ts_encoder.value = ascending ? ts_range.start : ts_range.end; + Slice ts_data = ts_encoder.encode(); + + CursorResult result = cursor->lower_bound_multivalue(key_data, ts_data, false); + + if (!ascending) { + if (result) { + result = cursor->to_current_prev_multi(false); + } else { + result = cursor->find(key_data, false); + if (result) { + cursor->to_current_last_multi(false); + } + } + } + + if (result) { + MoveOperation move_op = ascending ? MoveOperation::multi_currentkey_nextvalue : MoveOperation::multi_currentkey_prevvalue; + auto it = CursorValuesIterator::make(std::move(cursor), move_op); + if (ts_range.contains(*it)) { + return it; + } + } + + return {}; + } + + Timestamps exec_with_eager_begin(TKey key, TimestampRange ts_range, bool ascending) { + auto begin_it = begin(std::move(key), ts_range, ascending); + return std::ranges::subrange{std::move(begin_it), CursorValuesIterator{}} | + std::views::filter(ts_range.contains_predicate()); + } + + auto exec(TKey key, TimestampRange ts_range, bool ascending) { + auto exec_func = [query = *this, key = std::move(key), ts_range, ascending](std::monostate) mutable { + return query.exec_with_eager_begin(std::move(key), ts_range, ascending); + }; + // turn into a lazy view that runs exec_func only when iteration is started using range::begin() + return std::views::single(std::monostate{}) | std::views::transform(std::move(exec_func)) | std::views::join; + } +}; + +} // namespace silkworm::datastore::kvdb diff --git a/silkworm/db/datastore/kvdb/inverted_index_range_by_key_query_test.cpp b/silkworm/db/datastore/kvdb/inverted_index_range_by_key_query_test.cpp new file mode 100644 index 0000000000..e1ddf7a0ce --- /dev/null +++ b/silkworm/db/datastore/kvdb/inverted_index_range_by_key_query_test.cpp @@ -0,0 +1,108 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "inverted_index_range_by_key_query.hpp" + +#include +#include +#include + +#include + +#include "big_endian_codec.hpp" +#include "database.hpp" +#include "inverted_index_put_query.hpp" +#include "silkworm/infra/common/directories.hpp" + +namespace silkworm::datastore::kvdb { + +std::vector vector_from_range(auto range) { + std::vector results; + std::ranges::copy(range, std::back_inserter(results)); + return results; +} + +void init_inverted_index(RWTxn& tx, InvertedIndex ii, std::multimap kvs) { + InvertedIndexPutQuery put_query{tx, ii}; + for (auto& entry : kvs) { + put_query.exec(entry.first, entry.second, true); + } +} + +TEST_CASE("InvertedIndexRangeByKeyQuery") { + const TemporaryDirectory tmp_dir; + ::mdbx::env_managed env = open_env(EnvConfig{.path = tmp_dir.path(), .create = true, .in_memory = true}); + + EntityName name{"Test"}; + Schema::DatabaseDef schema; + schema.inverted_index(name); + + Database db{std::move(env), schema}; + db.create_tables(); + InvertedIndex ii = db.inverted_index(name); + RWAccess db_access = db.access_rw(); + + auto find_in = [&db_access, &ii](std::multimap kvs, uint64_t key, TimestampRange ts_range, bool ascending) -> std::vector { + { + RWTxnManaged tx = db_access.start_rw_tx(); + init_inverted_index(tx, ii, std::move(kvs)); + tx.commit_and_stop(); + } + + ROTxnManaged tx = db_access.start_ro_tx(); + InvertedIndexRangeByKeyQuery query{tx, ii}; + return vector_from_range(query.exec(key, ts_range, ascending)); + }; + + SECTION("asc - all") { + CHECK(find_in({{1, 1}, {1, 2}, {1, 3}}, 1, TimestampRange{0, 10}, true) == std::vector{1, 2, 3}); + } + SECTION("asc - all with neighbor keys") { + CHECK(find_in({{0, 2}, {1, 1}, {1, 2}, {1, 3}, {2, 2}}, 1, TimestampRange{0, 10}, true) == std::vector{1, 2, 3}); + } + SECTION("asc - middle") { + CHECK(find_in({{1, 1}, {1, 2}, {1, 3}}, 1, TimestampRange{2, 3}, true) == std::vector{2}); + } + SECTION("asc - middle gap") { + CHECK(find_in({{1, 1}, {1, 3}}, 1, TimestampRange{2, 3}, true).empty()); + } + SECTION("asc - middle to end with neighbor keys") { + CHECK(find_in({{0, 2}, {1, 1}, {1, 2}, {1, 3}, {2, 2}}, 1, TimestampRange{2, 10}, true) == std::vector{2, 3}); + } + SECTION("asc - middle gap to end with neighbor keys") { + CHECK(find_in({{0, 2}, {1, 1}, {1, 3}, {2, 2}}, 1, TimestampRange{2, 10}, true) == std::vector{3}); + } + SECTION("desc - all") { + CHECK(find_in({{1, 1}, {1, 2}, {1, 3}}, 1, TimestampRange{0, 10}, false) == std::vector{3, 2, 1}); + } + SECTION("desc - all with neighbor keys") { + CHECK(find_in({{0, 2}, {1, 1}, {1, 2}, {1, 3}, {2, 2}}, 1, TimestampRange{0, 10}, false) == std::vector{3, 2, 1}); + } + SECTION("desc - middle") { + CHECK(find_in({{1, 1}, {1, 2}, {1, 3}}, 1, TimestampRange{2, 3}, false) == std::vector{2}); + } + SECTION("desc - middle gap") { + CHECK(find_in({{1, 1}, {1, 3}}, 1, TimestampRange{2, 3}, false).empty()); + } + SECTION("desc - middle to start with neighbor keys") { + CHECK(find_in({{0, 2}, {1, 1}, {1, 2}, {1, 3}, {2, 2}}, 1, TimestampRange{0, 3}, false) == std::vector{2, 1}); + } + SECTION("desc - middle gap to start with neighbor keys") { + CHECK(find_in({{0, 2}, {1, 1}, {1, 3}, {2, 2}}, 1, TimestampRange{0, 3}, false) == std::vector{1}); + } +} + +} // namespace silkworm::datastore::kvdb diff --git a/silkworm/db/datastore/snapshots/common/util/iterator/map_values_view.hpp b/silkworm/db/datastore/snapshots/common/util/iterator/map_values_view.hpp index 70e213916a..1d44b9e2bc 100644 --- a/silkworm/db/datastore/snapshots/common/util/iterator/map_values_view.hpp +++ b/silkworm/db/datastore/snapshots/common/util/iterator/map_values_view.hpp @@ -24,7 +24,7 @@ namespace silkworm::map_values_view::fallback { template -class MapValuesView : std::ranges::view_interface> { +class MapValuesView : public std::ranges::view_interface> { public: using Map = std::map; diff --git a/silkworm/db/datastore/snapshots/inverted_index.hpp b/silkworm/db/datastore/snapshots/inverted_index.hpp index 6095ed1599..ec2b97940b 100644 --- a/silkworm/db/datastore/snapshots/inverted_index.hpp +++ b/silkworm/db/datastore/snapshots/inverted_index.hpp @@ -28,7 +28,7 @@ struct InvertedIndex { template segment::KVSegmentReader kv_segment_reader() { - return {kv_segment}; + return segment::KVSegmentReader{kv_segment}; } }; diff --git a/silkworm/db/datastore/snapshots/inverted_index_queries.hpp b/silkworm/db/datastore/snapshots/inverted_index_queries.hpp new file mode 100644 index 0000000000..5ac08e359e --- /dev/null +++ b/silkworm/db/datastore/snapshots/inverted_index_queries.hpp @@ -0,0 +1,19 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include "inverted_index_range_by_key_query.hpp" \ No newline at end of file diff --git a/silkworm/db/datastore/snapshots/inverted_index_range_by_key_query.hpp b/silkworm/db/datastore/snapshots/inverted_index_range_by_key_query.hpp new file mode 100644 index 0000000000..12e8dad0b0 --- /dev/null +++ b/silkworm/db/datastore/snapshots/inverted_index_range_by_key_query.hpp @@ -0,0 +1,108 @@ +/* + Copyright 2024 The Silkworm Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include + +#include "../common/owning_view.hpp" +#include "../common/timestamp.hpp" +#include "common/raw_codec.hpp" +#include "inverted_index.hpp" +#include "snapshot_bundle.hpp" +#include "snapshot_repository_ro_access.hpp" + +namespace silkworm::snapshots { + +template +auto timestamp_range_filter(elias_fano::EliasFanoList32 list, datastore::TimestampRange ts_range) { + if constexpr (ascending) { + return silkworm::ranges::owning_view(std::move(list)) | std::views::all | std::views::filter(ts_range.contains_predicate()); + } else { + return silkworm::ranges::owning_view(std::move(list)) | std::views::reverse | std::views::filter(ts_range.contains_predicate()); + } +} + +template +struct InvertedIndexFindByKeySegmentQuery { + explicit InvertedIndexFindByKeySegmentQuery( + InvertedIndex inverted_index) + : inverted_index_{inverted_index} {} + explicit InvertedIndexFindByKeySegmentQuery( + const SnapshotBundle& bundle, + datastore::EntityName inverted_index_name) + : inverted_index_{bundle.inverted_index(inverted_index_name)} {} + + using TKey = decltype(TKeyEncoder::value); + + std::optional exec(TKey key) { + TKeyEncoder key_encoder; + key_encoder.value = std::move(key); + ByteView key_data = key_encoder.encode_word(); + + auto offset = inverted_index_.accessor_index.lookup_by_key(key_data); + if (!offset) { + return std::nullopt; + } + + auto reader = inverted_index_.kv_segment_reader>(); + std::optional> result = reader.seek_one(*offset); + + // ensure that the found key matches to avoid lookup_by_key false positives + if (result && (result->first == key_data)) { + return std::move(result->second); + } + + return std::nullopt; + } + + template + auto exec_filter(TKey key, datastore::TimestampRange ts_range) { + return timestamp_range_filter(exec(std::move(key)).value_or(elias_fano::EliasFanoList32::empty_list()), ts_range); + } + + private: + InvertedIndex inverted_index_; +}; + +template +struct InvertedIndexRangeByKeyQuery { + explicit InvertedIndexRangeByKeyQuery( + const SnapshotRepositoryROAccess& repository, + datastore::EntityName inverted_index_name) + : repository_{repository}, + inverted_index_name_{inverted_index_name} {} + + using TKey = decltype(TKeyEncoder::value); + + template + auto exec(TKey key, datastore::TimestampRange ts_range) { + auto timestamps_in_bundle = [inverted_index_name = inverted_index_name_, key = std::move(key), ts_range](std::shared_ptr bundle) { + InvertedIndexFindByKeySegmentQuery query{*bundle, inverted_index_name}; + return query.template exec_filter(key, ts_range); + }; + + return silkworm::ranges::owning_view(repository_.bundles_intersecting_range(ts_range, ascending)) | + std::views::transform(std::move(timestamps_in_bundle)) | + std::views::join; + } + + private: + const SnapshotRepositoryROAccess& repository_; + datastore::EntityName inverted_index_name_; +}; + +} // namespace silkworm::snapshots diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.cpp b/silkworm/db/datastore/snapshots/snapshot_repository.cpp index 10e38ae622..0bcdcbafaf 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.cpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.cpp @@ -122,7 +122,7 @@ void SnapshotRepository::reopen_folder() { if (file_ranges.empty()) return; // sort file_ranges by range.start - std::sort(file_ranges.begin(), file_ranges.end(), [](const StepRange& r1, const StepRange& r2) -> bool { + std::ranges::sort(file_ranges, [](const StepRange& r1, const StepRange& r2) -> bool { return r1.start < r2.start; }); @@ -182,6 +182,30 @@ std::vector> SnapshotRepository::bundles_in_rang return bundles; } +std::vector> SnapshotRepository::bundles_intersecting_range(StepRange range, bool ascending) const { + if (range.size() == 0) { + return {}; + } + std::vector> bundles; + for (const auto& bundle : view_bundles()) { + StepRange bundle_range = bundle->step_range(); + if (range.contains_range(bundle_range) || bundle_range.contains(range.start) || bundle_range.contains(Step{range.end.value - 1})) { + bundles.push_back(bundle); + } + } + if (!ascending) { + std::ranges::reverse(bundles); + } + return bundles; +} + +std::vector> SnapshotRepository::bundles_intersecting_range(TimestampRange range, bool ascending) const { + if (range.size() == 0) { + return {}; + } + return bundles_intersecting_range(step_converter_->step_range_from_timestamp_range(range), ascending); +} + SnapshotPathList SnapshotRepository::get_files(std::string_view ext) const { ensure(fs::exists(dir_path_), [&]() { return "SnapshotRepository: " + dir_path_.string() + " does not exist"; }); @@ -204,7 +228,7 @@ SnapshotPathList SnapshotRepository::get_files(std::string_view ext) const { } // Order snapshot files by version/block-range/type - std::sort(snapshot_files.begin(), snapshot_files.end()); + std::ranges::sort(snapshot_files, std::less{}); return snapshot_files; } @@ -247,8 +271,8 @@ SnapshotPathList SnapshotRepository::stale_index_paths() const { SnapshotBundlePaths some_bundle_paths{schema_, path(), {Step{0}, Step{1}}}; auto accessor_index_file_ext = some_bundle_paths.accessor_index_paths().begin()->second.extension(); auto all_files = get_files(accessor_index_file_ext); - std::copy_if( - all_files.begin(), all_files.end(), + std::ranges::copy_if( + all_files, std::back_inserter(results), [this](const SnapshotPath& index_path) { return this->is_stale_index_path(index_path); }); return results; diff --git a/silkworm/db/datastore/snapshots/snapshot_repository.hpp b/silkworm/db/datastore/snapshots/snapshot_repository.hpp index c3fe889f77..46ad9d2e5d 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository.hpp @@ -96,6 +96,8 @@ class SnapshotRepository : public SnapshotRepositoryROAccess { std::shared_ptr find_bundle(Step step) const override; std::vector> bundles_in_range(StepRange range) const override; + std::vector> bundles_intersecting_range(StepRange range, bool ascending) const override; + std::vector> bundles_intersecting_range(TimestampRange range, bool ascending) const override; private: Step max_end_step() const; diff --git a/silkworm/db/datastore/snapshots/snapshot_repository_ro_access.hpp b/silkworm/db/datastore/snapshots/snapshot_repository_ro_access.hpp index 9ce366f3d1..307e5fa68d 100644 --- a/silkworm/db/datastore/snapshots/snapshot_repository_ro_access.hpp +++ b/silkworm/db/datastore/snapshots/snapshot_repository_ro_access.hpp @@ -35,6 +35,7 @@ struct SnapshotBundle; struct SnapshotRepositoryROAccess { using Timestamp = datastore::Timestamp; + using TimestampRange = datastore::TimestampRange; using Step = datastore::Step; using StepRange = datastore::StepRange; using Bundles = std::map>; @@ -74,7 +75,14 @@ struct SnapshotRepositoryROAccess { virtual std::shared_ptr find_bundle(Timestamp t) const = 0; virtual std::shared_ptr find_bundle(Step step) const = 0; + //! Bundles fully contained within a given range: range_start <= first_start < last_end <= range_end virtual std::vector> bundles_in_range(StepRange range) const = 0; + + //! Bundles having some steps within a given range: first_start <= range_start < range_end <= last_end + virtual std::vector> bundles_intersecting_range(StepRange range, bool ascending) const = 0; + + //! Bundles having some timestamps within a given range + virtual std::vector> bundles_intersecting_range(TimestampRange range, bool ascending) const = 0; }; } // namespace silkworm::snapshots diff --git a/silkworm/db/kv/api/local_transaction.cpp b/silkworm/db/kv/api/local_transaction.cpp index 65865e9b2a..40950761b8 100644 --- a/silkworm/db/kv/api/local_transaction.cpp +++ b/silkworm/db/kv/api/local_transaction.cpp @@ -17,9 +17,14 @@ #include "local_transaction.hpp" #include +#include +#include +#include namespace silkworm::db::kv::api { +using namespace silkworm::datastore; + Task LocalTransaction::open() { co_return; } @@ -83,7 +88,31 @@ Task LocalTransaction::history_seek(HistoryPointQuery /*quer co_return HistoryPointResult{}; } -Task LocalTransaction::index_range(IndexRangeQuery /*query*/) { +Task LocalTransaction::index_range(IndexRangeQuery query) { + // TODO: convert query.table to II EntityName + datastore::EntityName inverted_index_name = state::kInvIdxNameLogAddress; + InvertedIndexRangeByKeyQuery, snapshots::RawEncoder> store_query{ + inverted_index_name, + data_store_.chaindata, + txn_, + data_store_.state_repository, + }; + + // TODO: convert query from/to to ts_range + auto ts_range = datastore::TimestampRange{0, 10}; + size_t limit = (query.limit == kUnlimited) ? std::numeric_limits::max() : static_cast(query.limit); + + if (query.ascending_order) { + // TODO: this is just a test example, instead of direct iteration, apply page_size using std::views::chunk, + // TODO: save the range for future requests using page_token and return the first chunk + for ([[maybe_unused]] datastore::Timestamp t : store_query.exec(query.key, ts_range) | std::views::take(limit)) { + } + } else { + // TODO: same, this is just a test example + for ([[maybe_unused]] datastore::Timestamp t : store_query.exec(query.key, ts_range) | std::views::take(limit)) { + } + } + // TODO(canepat) implement using E3-like aggregator abstraction [tx_id_ must be changed] auto paginator = [](api::PaginatedTimestamps::PageToken) mutable -> Task { co_return api::PaginatedTimestamps::PageResult{};