Skip to content

Commit

Permalink
db: datastore layer (#2416)
Browse files Browse the repository at this point in the history
  • Loading branch information
@battlmonstr
battlmonstr authored Oct 10, 2024
1 parent 01c8b72 commit 1821e07
Show file tree
Hide file tree
Showing 301 changed files with 2,815 additions and 2,707 deletions.
2 changes: 1 addition & 1 deletion cmake/copyright.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ list(FILTER SRC EXCLUDE REGEX [[silkworm/core/common/lru_cache(_test)?\..pp$]])
list(FILTER SRC EXCLUDE REGEX [[silkworm/core/crypto/kzg\.cpp$]])
list(FILTER SRC EXCLUDE REGEX [[silkworm/infra/concurrency/thread_pool\.hpp$]])
list(FILTER SRC EXCLUDE REGEX [[silkworm/interfaces/]])
list(FILTER SRC EXCLUDE REGEX [[silkworm/db/snapshots/config/[a-z_]+\.hpp$]])
list(FILTER SRC EXCLUDE REGEX [[silkworm/db/datastore/snapshots/config/[a-z_]+\.hpp$]])
list(FILTER SRC EXCLUDE REGEX [[silkworm/rpc/json_rpc/specification\.cpp$]])
list(FILTER SRC EXCLUDE REGEX [[silkworm/sync/internals/preverified_hashes/preverified_hashes_[a-z]+\.cpp$]])

Expand Down
4 changes: 2 additions & 2 deletions cmd/capi/execute.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,9 @@

#include <silkworm/capi/silkworm.h>
#include <silkworm/db/access_layer.hpp>
#include <silkworm/db/mdbx/mdbx.hpp>
#include <silkworm/db/datastore/mdbx/mdbx.hpp>
#include <silkworm/db/datastore/snapshots/snapshot_repository.hpp>
#include <silkworm/db/snapshot_bundle_factory_impl.hpp>
#include <silkworm/db/snapshots/snapshot_repository.hpp>
#include <silkworm/infra/common/directories.hpp>
#include <silkworm/infra/common/log.hpp>
#include <silkworm/rpc/daemon.hpp>
Expand Down
2 changes: 1 addition & 1 deletion cmd/common/db_max_readers_option.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

#include "db_max_readers_option.hpp"

#include <silkworm/db/mdbx/mdbx.hpp>
#include <silkworm/db/datastore/mdbx/mdbx.hpp>

namespace silkworm::cmd::common {

Expand Down
2 changes: 1 addition & 1 deletion cmd/common/snapshot_options.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

#include <CLI/CLI.hpp>

#include <silkworm/db/snapshots/snapshot_settings.hpp>
#include <silkworm/db/datastore/snapshots/snapshot_settings.hpp>

namespace silkworm::cmd::common {

Expand Down
2 changes: 1 addition & 1 deletion cmd/dev/backend_kv_server.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
#include <silkworm/core/types/evmc_bytes32.hpp>
#include <silkworm/db/access_layer.hpp>
#include <silkworm/db/chain_head.hpp>
#include <silkworm/db/mdbx/mdbx.hpp>
#include <silkworm/db/datastore/mdbx/mdbx.hpp>
#include <silkworm/infra/common/directories.hpp>
#include <silkworm/infra/common/log.hpp>
#include <silkworm/infra/concurrency/awaitable_wait_for_one.hpp>
Expand Down
2 changes: 1 addition & 1 deletion cmd/dev/check_changes.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@
#include <silkworm/core/types/evmc_bytes32.hpp>
#include <silkworm/db/access_layer.hpp>
#include <silkworm/db/buffer.hpp>
#include <silkworm/db/datastore/snapshots/snapshot_repository.hpp>
#include <silkworm/db/snapshot_bundle_factory_impl.hpp>
#include <silkworm/db/snapshots/snapshot_repository.hpp>
#include <silkworm/infra/common/directories.hpp>
#include <silkworm/infra/common/log.hpp>
#include <silkworm/infra/concurrency/signal_handler.hpp>
Expand Down
47 changes: 24 additions & 23 deletions cmd/dev/check_hashstate.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,32 +24,33 @@
#include <silkworm/infra/common/log.hpp>

using namespace silkworm;
using namespace silkworm::db;

enum Operation {
kHashAccount,
kHashStorage,
kCode,
};

std::pair<db::MapConfig, db::MapConfig> get_tables_for_checking(Operation operation) {
std::pair<MapConfig, MapConfig> get_tables_for_checking(Operation operation) {
switch (operation) {
case kHashAccount:
return {db::table::kPlainState, db::table::kHashedAccounts};
return {table::kPlainState, table::kHashedAccounts};
case kHashStorage:
return {db::table::kPlainState, db::table::kHashedStorage};
return {table::kPlainState, table::kHashedStorage};
default:
return {db::table::kPlainCodeHash, db::table::kHashedCodeHash};
return {table::kPlainCodeHash, table::kHashedCodeHash};
}
}

void check(mdbx::txn& txn, Operation operation) {
auto [source_config, target_config] = get_tables_for_checking(operation);
auto source_table{db::open_cursor(txn, source_config)};
auto target_table{db::open_cursor(txn, target_config)};
auto source_table{open_cursor(txn, source_config)};
auto target_table{open_cursor(txn, target_config)};
auto data{source_table.to_first(/*throw_notfound*/ false)};

while (data) { /* Loop as long as we have no errors*/
Bytes mdb_key_as_bytes{db::from_slice(data.key)};
Bytes mdb_key_as_bytes{from_slice(data.key)};

if (operation == kHashAccount) {
// Account
Expand All @@ -60,14 +61,14 @@ void check(mdbx::txn& txn, Operation operation) {
auto hash{keccak256(mdb_key_as_bytes)};
ByteView key{hash.bytes};

auto actual_value{target_table.find(db::to_slice(key))};
auto actual_value{target_table.find(to_slice(key))};
if (!actual_value) {
log::Error() << "key: " << to_hex(key) << ", does not exist.";
return;
}
if (actual_value.value != data.value) {
log::Error() << "Expected: " << to_hex(db::from_slice(data.value)) << ", Actual: << "
<< to_hex(db::from_slice(actual_value.value));
log::Error() << "Expected: " << to_hex(from_slice(data.value)) << ", Actual: << "
<< to_hex(from_slice(actual_value.value));
return;
}
data = source_table.to_next(false);
Expand All @@ -79,13 +80,13 @@ void check(mdbx::txn& txn, Operation operation) {
continue;
}

Bytes key(kHashLength * 2 + db::kIncarnationLength, '\0');
Bytes key(kHashLength * 2 + kIncarnationLength, '\0');
std::memcpy(&key[0], keccak256(mdb_key_as_bytes.substr(0, kAddressLength)).bytes, kHashLength);
std::memcpy(&key[kHashLength], &mdb_key_as_bytes[kAddressLength], db::kIncarnationLength);
std::memcpy(&key[kHashLength + db::kIncarnationLength],
keccak256(mdb_key_as_bytes.substr(kAddressLength + db::kIncarnationLength)).bytes, kHashLength);
std::memcpy(&key[kHashLength], &mdb_key_as_bytes[kAddressLength], kIncarnationLength);
std::memcpy(&key[kHashLength + kIncarnationLength],
keccak256(mdb_key_as_bytes.substr(kAddressLength + kIncarnationLength)).bytes, kHashLength);

auto target_data{target_table.find_multivalue(db::to_slice(key), data.value, /*throw_notfound*/ false)};
auto target_data{target_table.find_multivalue(to_slice(key), data.value, /*throw_notfound*/ false)};
if (!target_data) {
log::Error() << "Key: " << to_hex(key) << ", does not exist.";
return;
Expand All @@ -94,22 +95,22 @@ void check(mdbx::txn& txn, Operation operation) {

} else {
// Code
if (data.key.length() != kAddressLength + db::kIncarnationLength) {
if (data.key.length() != kAddressLength + kIncarnationLength) {
data = source_table.to_next(false);
continue;
}
Bytes key(kHashLength + db::kIncarnationLength, '\0');
Bytes key(kHashLength + kIncarnationLength, '\0');
std::memcpy(&key[0], keccak256(mdb_key_as_bytes.substr(0, kAddressLength)).bytes, kHashLength);
std::memcpy(&key[kHashLength], &mdb_key_as_bytes[kAddressLength], db::kIncarnationLength);
auto actual_value{target_table.find(db::to_slice(key), /*throw_notfound*/ false)};
std::memcpy(&key[kHashLength], &mdb_key_as_bytes[kAddressLength], kIncarnationLength);
auto actual_value{target_table.find(to_slice(key), /*throw_notfound*/ false)};
if (!actual_value) {
log::Error() << "Key: " << to_hex(key) << ", does not exist.";
data = source_table.to_next(false);
continue;
}
if (actual_value.value != data.value) {
log::Error() << "Expected: " << to_hex(db::from_slice(data.value)) << ", Actual: << "
<< to_hex(db::from_slice(actual_value.value));
log::Error() << "Expected: " << to_hex(from_slice(data.value)) << ", Actual: << "
<< to_hex(from_slice(actual_value.value));
return;
}
data = source_table.to_next(false);
Expand All @@ -130,8 +131,8 @@ int main(int argc, char* argv[]) {
try {
auto data_dir{DataDirectory::from_chaindata(chaindata)};
data_dir.deploy();
db::EnvConfig db_config{data_dir.chaindata().path().string()};
auto env{db::open_env(db_config)};
EnvConfig db_config{data_dir.chaindata().path().string()};
auto env{open_env(db_config)};
auto txn{env.start_write()};

log::Info() << "Checking Accounts";
Expand Down
39 changes: 20 additions & 19 deletions cmd/dev/check_log_indices.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@
#include <silkworm/core/common/bytes_to_string.hpp>
#include <silkworm/core/types/address.hpp>
#include <silkworm/core/types/evmc_bytes32.hpp>
#include <silkworm/db/datastore/mdbx/bitmap.hpp>
#include <silkworm/db/datastore/mdbx/mdbx.hpp>
#include <silkworm/db/log_cbor.hpp>
#include <silkworm/db/mdbx/bitmap.hpp>
#include <silkworm/db/mdbx/mdbx.hpp>
#include <silkworm/db/tables.hpp>
#include <silkworm/db/util.hpp>
#include <silkworm/infra/common/directories.hpp>
Expand All @@ -43,6 +43,7 @@

using Roaring = roaring::Roaring;
using namespace silkworm;
using namespace silkworm::db;
using namespace silkworm::cmd::common;

enum class TargetIndex {
Expand Down Expand Up @@ -137,38 +138,38 @@ void trace(const Log& log) {
}
}

void check_address_index(BlockNum block_number, const evmc::address& log_address, db::ROCursor* log_address_cursor) {
void check_address_index(BlockNum block_number, const evmc::address& log_address, ROCursor* log_address_cursor) {
// Transaction log address must be present in LogAddressIndex table
const auto log_address_key{db::log_address_key(log_address, block_number)};
const auto log_address_data{log_address_cursor->lower_bound(db::to_slice(log_address_key), false)};
const auto log_address_data{log_address_cursor->lower_bound(to_slice(log_address_key), false)};
ensure(log_address_data.done, [&]() { return "LogAddressIndex does not contain key " + to_hex(log_address_key); });

const auto [address_view, address_upper_bound_block] = db::split_log_address_key(log_address_data.key);
const auto [address_view, address_upper_bound_block] = split_log_address_key(log_address_data.key);
const auto& address_view_ref = address_view;
ensure(to_hex(address_view) == to_hex(log_address.bytes), [&]() { return "address mismatch in LogAddressIndex table: " + to_hex(address_view_ref); });
ensure(address_upper_bound_block >= block_number, [&]() { return "upper bound mismatch in LogAddressIndex table: " + to_hex(address_view_ref); });

// Retrieved chunk of the address roaring bitmap must contain the transaction log block
const auto& log_address_value{log_address_data.value};
const auto address_bitmap_chunk{db::bitmap::parse32(log_address_value)};
const auto address_bitmap_chunk{bitmap::parse32(log_address_value)};
ensure(address_bitmap_chunk.contains(static_cast<uint32_t>(block_number)),
[&]() { return "address bitmap chunk " + address_bitmap_chunk.toString() + " does not contain block " + std::to_string(block_number); });
}

void check_topic_index(BlockNum block_number, const evmc::bytes32& log_topic, db::ROCursor* log_topic_cursor) {
void check_topic_index(BlockNum block_number, const evmc::bytes32& log_topic, ROCursor* log_topic_cursor) {
// Each transaction log topic must be present in LogTopicIndex table
const auto log_topic_key{db::log_topic_key(log_topic, block_number)};
const auto log_topic_data{log_topic_cursor->lower_bound(db::to_slice(log_topic_key), false)};
const auto log_topic_data{log_topic_cursor->lower_bound(to_slice(log_topic_key), false)};
ensure(log_topic_data.done, [&]() { return "LogTopicIndex does not contain key " + to_hex(log_topic_key); });

const auto [topic_view, topic_upper_bound_block] = db::split_log_topic_key(log_topic_data.key);
const auto [topic_view, topic_upper_bound_block] = split_log_topic_key(log_topic_data.key);
const auto& topic_view_ref = topic_view;
ensure(to_hex(topic_view) == to_hex(log_topic.bytes), [&]() { return "topic mismatch in LogTopicIndex table: " + to_hex(topic_view_ref); });
ensure(topic_upper_bound_block >= block_number, [&]() { return "upper bound mismatch in LogTopicIndex table: " + to_hex(topic_view_ref); });

// Retrieved chunk of the topic roaring bitmap must contain the transaction log block
const auto& log_topic_value{log_topic_data.value};
const auto topic_bitmap_chunk{db::bitmap::parse32(log_topic_value)};
const auto topic_bitmap_chunk{bitmap::parse32(log_topic_value)};
ensure(topic_bitmap_chunk.contains(static_cast<uint32_t>(block_number)),
[&]() { return "topic bitmap chunk " + topic_bitmap_chunk.toString() + " does not contain block " + std::to_string(block_number); });
}
Expand Down Expand Up @@ -197,22 +198,22 @@ int main(int argc, char* argv[]) {
// Open the database and create a read-only txn
auto data_dir{DataDirectory::from_chaindata(settings.chaindata)};
data_dir.deploy();
db::EnvConfig db_config{data_dir.chaindata().path().string()};
auto env{db::open_env(db_config)};
db::ROTxnManaged txn{env};
EnvConfig db_config{data_dir.chaindata().path().string()};
auto env{open_env(db_config)};
ROTxnManaged txn{env};

auto logs_cursor = txn.ro_cursor(db::table::kLogs);
auto log_address_cursor = txn.ro_cursor(db::table::kLogAddressIndex);
auto log_topic_cursor = txn.ro_cursor(db::table::kLogTopicIndex);
auto logs_cursor = txn.ro_cursor(table::kLogs);
auto log_address_cursor = txn.ro_cursor(table::kLogAddressIndex);
auto log_topic_cursor = txn.ro_cursor(table::kLogTopicIndex);

log::Info() << "Check transaction log indices for blocks " << block_range(settings) << " ...";

// Start from the key having block_from as key prefix and iterate over TransactionLog on all blocks up to block_to
auto start_key_prefix{db::block_key(settings.block_from)};
auto logs_data{logs_cursor->lower_bound(db::to_slice(start_key_prefix), false)};
auto start_key_prefix{block_key(settings.block_from)};
auto logs_data{logs_cursor->lower_bound(to_slice(start_key_prefix), false)};
ensure(logs_data.done, "Nonexistent block range: block_from not found");
while (logs_data.done) {
const auto [block_number, tx_id] = db::split_log_key(logs_data.key);
const auto [block_number, tx_id] = split_log_key(logs_data.key);
if (settings.block_to && block_number > *settings.block_to) {
log::Info() << "Target block " << *settings.block_to << " reached";
break;
Expand Down
39 changes: 20 additions & 19 deletions cmd/dev/check_senders.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
#include "../common/common.hpp"

using namespace silkworm;
using namespace silkworm::db;
using namespace silkworm::cmd::common;

int main(int argc, char* argv[]) {
Expand Down Expand Up @@ -64,15 +65,15 @@ int main(int argc, char* argv[]) {

auto data_dir{DataDirectory::from_chaindata(chaindata)};
data_dir.deploy();
db::EnvConfig db_config{data_dir.chaindata().path().string()};
EnvConfig db_config{data_dir.chaindata().path().string()};

auto env{db::open_env(db_config)};
db::ROTxnManaged txn{env};
auto env{open_env(db_config)};
ROTxnManaged txn{env};

auto canonical_hashes_cursor = txn.ro_cursor(db::table::kCanonicalHashes);
auto bodies_cursor = txn.ro_cursor(db::table::kBlockBodies);
auto tx_cursor = txn.ro_cursor(db::table::kBlockTransactions);
auto senders_cursor = txn.ro_cursor(db::table::kSenders);
auto canonical_hashes_cursor = txn.ro_cursor(table::kCanonicalHashes);
auto bodies_cursor = txn.ro_cursor(table::kBlockBodies);
auto tx_cursor = txn.ro_cursor(table::kBlockTransactions);
auto senders_cursor = txn.ro_cursor(table::kSenders);

uint64_t expected_block_number{block_from};
uint64_t processed_senders_count{0};
Expand All @@ -83,9 +84,9 @@ int main(int argc, char* argv[]) {
// Seek at the first block body (if any)
Bytes first_block(8, '\0');
endian::store_big_u64(first_block.data(), block_from);
const bool block_from_found = bodies_cursor->seek(db::to_slice(first_block));
const bool block_from_found = bodies_cursor->seek(to_slice(first_block));
if (block_from_found) {
log::Error() << "First block " << block_from << " not found in " << db::table::kBlockBodies.name << " table";
log::Error() << "First block " << block_from << " not found in " << table::kBlockBodies.name << " table";
return -1;
}

Expand All @@ -100,28 +101,28 @@ int main(int argc, char* argv[]) {
}

// Decode block body data as RLP buffer
auto body_rlp{db::from_slice(bodies_data.value)};
auto body_rlp{from_slice(bodies_data.value)};
auto body{unwrap_or_throw(decode_stored_block_body(body_rlp))};

// Process block transactions one-by-one
log::Debug() << "Processing block: " << block_number << " txn count: " << body.txn_count;
if (body.txn_count > 0) {
// Retrieve canonical block hash
const Bytes canonical_key{db::block_key(block_number)};
const auto canonical_data{canonical_hashes_cursor->find(db::to_slice(canonical_key), false)};
const Bytes canonical_key{block_key(block_number)};
const auto canonical_data{canonical_hashes_cursor->find(to_slice(canonical_key), false)};
if (!canonical_data) {
log::Error() << "Block " << block_number << " not found in " << db::table::kCanonicalHashes.name << " table";
log::Error() << "Block " << block_number << " not found in " << table::kCanonicalHashes.name << " table";
continue;
}
SILKWORM_ASSERT(canonical_data.value.length() == kHashLength);
auto block_hash = to_bytes32({static_cast<const uint8_t*>(canonical_data.value.data()), kHashLength});
log::Debug() << "Block hash: " << to_hex(block_hash);

// Read the ordered sequence of block senders (one for each transaction)
auto senders_key{db::block_key(block_number, block_hash.bytes)};
auto senders_data{senders_cursor->find(db::to_slice(senders_key), /*throw_notfound = */ false)};
auto senders_key{block_key(block_number, block_hash.bytes)};
auto senders_data{senders_cursor->find(to_slice(senders_key), /*throw_notfound = */ false)};
if (!senders_data) {
log::Error() << "Block " << block_number << " hash " << to_hex(block_hash) << " not found in " << db::table::kSenders.name << " table";
log::Error() << "Block " << block_number << " hash " << to_hex(block_hash) << " not found in " << table::kSenders.name << " table";
break;
}

Expand All @@ -139,13 +140,13 @@ int main(int argc, char* argv[]) {
// Read block transactions one at a time
std::vector<Transaction> transactions;
uint64_t i{0};
auto tx_data{tx_cursor->find(db::to_slice(tx_key), false)};
auto tx_data{tx_cursor->find(to_slice(tx_key), false)};
for (; i < body.txn_count && tx_data.done; ++i, tx_data = tx_cursor->to_next(false)) {
if (!tx_data) {
log::Error() << "Block " << block_number << " tx " << i << " not found in " << db::table::kBlockTransactions.name << " table";
log::Error() << "Block " << block_number << " tx " << i << " not found in " << table::kBlockTransactions.name << " table";
continue;
}
ByteView transaction_rlp{db::from_slice(tx_data.value)};
ByteView transaction_rlp{from_slice(tx_data.value)};

// Decode transaction data as RLP buffer
Transaction tx;
Expand Down
Loading

0 comments on commit 1821e07

Please sign in to comment.