diff --git a/cloud/blockstore/libs/.gitignore b/cloud/blockstore/libs/.gitignore index 0bb362c2496..78d43add86b 100644 --- a/cloud/blockstore/libs/.gitignore +++ b/cloud/blockstore/libs/.gitignore @@ -41,6 +41,7 @@ storage/disk_agent/ut/cloud-blockstore-libs-storage-disk_agent-ut storage/disk_registry_proxy/ut/blockstore-libs-storage-disk_registry_proxy-ut storage/disk_registry/actors/ut/libs-storage-disk_registry-actors-ut storage/disk_registry/model/ut/blockstore-libs-storage-disk_registry-model-ut +storage/disk_registry/benchmark/benchmark storage/disk_registry/ut_allocation/libs-storage-disk_registry-ut_allocation storage/disk_registry/ut_checkpoint/libs-storage-disk_registry-ut_checkpoint storage/disk_registry/ut_cms/blockstore-libs-storage-disk_registry-ut_cms diff --git a/cloud/blockstore/libs/storage/disk_registry/benchmark/ya.make b/cloud/blockstore/libs/storage/disk_registry/benchmark/ya.make new file mode 100644 index 00000000000..3c732ec9637 --- /dev/null +++ b/cloud/blockstore/libs/storage/disk_registry/benchmark/ya.make @@ -0,0 +1,19 @@ +G_BENCHMARK() + +INCLUDE(${ARCADIA_ROOT}/cloud/blockstore/tests/recipes/disk-registry-state/recipe.inc) + +IF (SANITIZER_TYPE) + INCLUDE(${ARCADIA_ROOT}/cloud/storage/core/tests/recipes/large.inc) +ELSE() + INCLUDE(${ARCADIA_ROOT}/cloud/storage/core/tests/recipes/medium.inc) +ENDIF() + +SRCS( + ../disk_registry_state_benchmark.cpp +) + +PEERDIR( + cloud/blockstore/libs/storage/disk_registry/testlib +) + +END() diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor.h b/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor.h index 32a19889b3a..a295b3b1342 100644 --- a/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor.h +++ b/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor.h @@ -502,6 +502,8 @@ class TDiskRegistryActor final } \ // BLOCKSTORE_DISK_REGISTRY_COUNTER +TDiskRegistryStateSnapshot MakeNewLoadState( + NProto::TDiskRegistryStateBackup&& backup); bool ToLogicalBlocks(NProto::TDeviceConfig& device, ui32 logicalBlockSize); TString LogDevices(const TVector& devices); diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor_restore_state.cpp b/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor_restore_state.cpp index 5f818f502f0..a74b2b62c01 100644 --- a/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor_restore_state.cpp +++ b/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor_restore_state.cpp @@ -15,120 +15,6 @@ namespace { //////////////////////////////////////////////////////////////////////////////// -TDiskRegistryStateSnapshot MakeNewLoadState( - NProto::TDiskRegistryStateBackup&& backup) -{ - auto move = [] (auto& src, auto& dst) { - dst.reserve(src.size()); - dst.assign( - std::make_move_iterator(src.begin()), - std::make_move_iterator(src.end())); - src.Clear(); - }; - - auto transform = [] (auto& src, auto& dst, auto func) { - dst.resize(src.size()); - for (int i = 0; i < src.size(); ++i) { - func(src[i], dst[i]); - } - src.Clear(); - }; - - TDiskRegistryStateSnapshot newLoadState; - // if new fields are added to TDiskRegistryStateSnapshot - // there will be a compilation error. - auto& [ - config, - dirtyDevices, - agents, - disks, - placementGroups, - brokenDisks, - disksToReallocate, - diskStateChanges, - lastDiskStateSeqNo, - writableState, - disksToCleanup, - errorNotifications, - userNotifications, - outdatedVolumeConfigs, - suspendedDevices, - automaticallyReplacedDevices, - diskRegistryAgentListParams - ] = newLoadState; - - if (backup.DirtyDevicesSize()) { - transform( - *backup.MutableDirtyDevices(), - dirtyDevices, - [] (auto& src, auto& dst) { - dst.Id = src.GetId(); - dst.DiskId = src.GetDiskId(); - }); - } else { - transform( - *backup.MutableOldDirtyDevices(), - dirtyDevices, - [] (auto& src, auto& dst) { - dst.Id = src; - }); - } - - move(*backup.MutableAgents(), agents); - move(*backup.MutableDisks(), disks); - move(*backup.MutablePlacementGroups(), placementGroups); - move(*backup.MutableDisksToNotify(), disksToReallocate); - move(*backup.MutableDisksToCleanup(), disksToCleanup); - - move(*backup.MutableErrorNotifications(), errorNotifications); - move(*backup.MutableUserNotifications(), userNotifications); - // Filter out unknown events for future version rollback compatibility - std::erase_if(userNotifications, [] (const auto& notif) { - return notif.GetEventCase() - == NProto::TUserNotification::EventCase::EVENT_NOT_SET; - }); - - move(*backup.MutableOutdatedVolumeConfigs(), outdatedVolumeConfigs); - move(*backup.MutableSuspendedDevices(), suspendedDevices); - - transform( - *backup.MutableBrokenDisks(), - brokenDisks, - [] (auto& src, auto& dst) { - dst.DiskId = src.GetDiskId(); - dst.TsToDestroy = TInstant::MicroSeconds(src.GetTsToDestroy()); - }); - transform( - *backup.MutableDiskStateChanges(), - diskStateChanges, - [] (auto& src, auto& dst) { - if (src.HasState()) { - dst.State.Swap(src.MutableState()); - } - dst.SeqNo = src.GetSeqNo(); - }); - transform( - *backup.MutableAutomaticallyReplacedDevices(), - automaticallyReplacedDevices, - [] (auto& src, auto& dst) { - dst.DeviceId = src.GetDeviceId(); - dst.ReplacementTs = TInstant::MicroSeconds(src.GetReplacementTs()); - }); - - diskRegistryAgentListParams.insert( - backup.MutableDiskRegistryAgentListParams()->begin(), - backup.MutableDiskRegistryAgentListParams()->end()); - - if (backup.HasConfig()) { - config.Swap(backup.MutableConfig()); - } - - lastDiskStateSeqNo = config.GetLastDiskStateSeqNo(); - writableState = config.GetWritableState(); - - return newLoadState; -} - using TOperations = TQueue>; void RestoreConfig( @@ -440,6 +326,122 @@ void RestoreDiskRegistryAgentListParams( //////////////////////////////////////////////////////////////////////////////// +TDiskRegistryStateSnapshot MakeNewLoadState( + NProto::TDiskRegistryStateBackup&& backup) +{ + auto move = [] (auto& src, auto& dst) { + dst.reserve(src.size()); + dst.assign( + std::make_move_iterator(src.begin()), + std::make_move_iterator(src.end())); + src.Clear(); + }; + + auto transform = [] (auto& src, auto& dst, auto func) { + dst.resize(src.size()); + for (int i = 0; i < src.size(); ++i) { + func(src[i], dst[i]); + } + src.Clear(); + }; + + TDiskRegistryStateSnapshot newLoadState; + // if new fields are added to TDiskRegistryStateSnapshot + // there will be a compilation error. + auto& [ + config, + dirtyDevices, + agents, + disks, + placementGroups, + brokenDisks, + disksToReallocate, + diskStateChanges, + lastDiskStateSeqNo, + writableState, + disksToCleanup, + errorNotifications, + userNotifications, + outdatedVolumeConfigs, + suspendedDevices, + automaticallyReplacedDevices, + diskRegistryAgentListParams + ] = newLoadState; + + if (backup.DirtyDevicesSize()) { + transform( + *backup.MutableDirtyDevices(), + dirtyDevices, + [] (auto& src, auto& dst) { + dst.Id = src.GetId(); + dst.DiskId = src.GetDiskId(); + }); + } else { + transform( + *backup.MutableOldDirtyDevices(), + dirtyDevices, + [] (auto& src, auto& dst) { + dst.Id = src; + }); + } + + move(*backup.MutableAgents(), agents); + move(*backup.MutableDisks(), disks); + move(*backup.MutablePlacementGroups(), placementGroups); + move(*backup.MutableDisksToNotify(), disksToReallocate); + move(*backup.MutableDisksToCleanup(), disksToCleanup); + + move(*backup.MutableErrorNotifications(), errorNotifications); + move(*backup.MutableUserNotifications(), userNotifications); + // Filter out unknown events for future version rollback compatibility + std::erase_if(userNotifications, [] (const auto& notif) { + return notif.GetEventCase() + == NProto::TUserNotification::EventCase::EVENT_NOT_SET; + }); + + move(*backup.MutableOutdatedVolumeConfigs(), outdatedVolumeConfigs); + move(*backup.MutableSuspendedDevices(), suspendedDevices); + + transform( + *backup.MutableBrokenDisks(), + brokenDisks, + [] (auto& src, auto& dst) { + dst.DiskId = src.GetDiskId(); + dst.TsToDestroy = TInstant::MicroSeconds(src.GetTsToDestroy()); + }); + transform( + *backup.MutableDiskStateChanges(), + diskStateChanges, + [] (auto& src, auto& dst) { + if (src.HasState()) { + dst.State.Swap(src.MutableState()); + } + dst.SeqNo = src.GetSeqNo(); + }); + transform( + *backup.MutableAutomaticallyReplacedDevices(), + automaticallyReplacedDevices, + [] (auto& src, auto& dst) { + dst.DeviceId = src.GetDeviceId(); + dst.ReplacementTs = TInstant::MicroSeconds(src.GetReplacementTs()); + }); + + diskRegistryAgentListParams.insert( + backup.MutableDiskRegistryAgentListParams()->begin(), + backup.MutableDiskRegistryAgentListParams()->end()); + + if (backup.HasConfig()) { + config.Swap(backup.MutableConfig()); + } + + lastDiskStateSeqNo = config.GetLastDiskStateSeqNo(); + writableState = config.GetWritableState(); + + return newLoadState; +} + +//////////////////////////////////////////////////////////////////////////////// + void TDiskRegistryActor::HandleRestoreDiskRegistryState( const TEvDiskRegistry::TEvRestoreDiskRegistryStateRequest::TPtr& ev, const TActorContext& ctx) diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_state_benchmark.cpp b/cloud/blockstore/libs/storage/disk_registry/disk_registry_state_benchmark.cpp new file mode 100644 index 00000000000..49c39407749 --- /dev/null +++ b/cloud/blockstore/libs/storage/disk_registry/disk_registry_state_benchmark.cpp @@ -0,0 +1,177 @@ +#include "disk_registry_state.h" + +#include +#include + +#include +#include + +#include + +#include + +namespace NCloud::NBlockStore::NStorage { + +namespace { + +/////////////////////////////////////////////////////////////////////////////// + +const TString BackupFile = + "cloud/blockstore/tests/recipes/disk-registry-state/data/backup.json"; + +TString BackupFilePath() +{ + auto arcRoot = ArcadiaSourceRoot(); + Y_ABORT_UNLESS(arcRoot); + return JoinFsPaths(arcRoot, BackupFile); +} + +TString ReadFile(const TString& filePath) +{ + TFile file(filePath, EOpenModeFlag::RdOnly); + return TFileInput(file).ReadAll(); +} + +TDiskRegistryState Load(bool disableFullGroupsCalc) +{ + auto backupPath = BackupFilePath(); + TString backupData = ReadFile(backupPath); + + NProto::TBackupDiskRegistryStateResponse backup; + + auto status = + google::protobuf::util::JsonStringToMessage(backupData, &backup); + if (!status.ok()) { + ythrow yexception() + << "Error loading state from: " << backupPath.Quote() + << " error: " << status.ToString(); + } + auto monitoring = CreateMonitoringServiceStub(); + auto diskRegistryGroup = monitoring->GetCounters() + ->GetSubgroup("counters", "blockstore") + ->GetSubgroup("component", "disk_registry"); + + auto snapshot = MakeNewLoadState(std::move(*backup.MutableBackup())); + + auto storeageConfigProto = + NDiskRegistryStateTest::CreateDefaultStorageConfigProto(); + storeageConfigProto.SetDisableFullPlacementGroupCountCalculation( + disableFullGroupsCalc); + storeageConfigProto.SetAllocationUnitNonReplicatedSSD(93); + + return TDiskRegistryState( + CreateLoggingService("console"), + NDiskRegistryStateTest::CreateStorageConfig( + std::move(storeageConfigProto)), + diskRegistryGroup, + std::move(snapshot.Config), + std::move(snapshot.Agents), + std::move(snapshot.Disks), + std::move(snapshot.PlacementGroups), + std::move(snapshot.BrokenDisks), + std::move(snapshot.DisksToReallocate), + std::move(snapshot.DiskStateChanges), + snapshot.LastDiskStateSeqNo, + std::move(snapshot.DirtyDevices), + std::move(snapshot.DisksToCleanup), + std::move(snapshot.ErrorNotifications), + std::move(snapshot.UserNotifications), + std::move(snapshot.OutdatedVolumeConfigs), + std::move(snapshot.SuspendedDevices), + std::move(snapshot.AutomaticallyReplacedDevices), + std::move(snapshot.DiskRegistryAgentListParams)); +} + +} // namespace + +static void PublishCounters_All(benchmark::State& benchmarkState) +{ + auto state = Load(false); + for (const auto _: benchmarkState) { + state.PublishCounters(TInstant::Now()); + } +} + +static void PublishCounters_DisableFullGroups(benchmark::State& benchmarkState) +{ + auto state = Load(true); + for (const auto _: benchmarkState) { + state.PublishCounters(TInstant::Now()); + } +} + +void DoCreateDeleteDisk( + benchmark::State& benchmarkState, + const TDiskRegistryState::TAllocateDiskParams& diskParams) +{ + TTestExecutor executor; + executor.WriteTx([&](TDiskRegistryDatabase db) mutable + { db.InitSchema(); }); + + auto state = Load(false); + for (const auto _: benchmarkState) { + executor.WriteTx( + [&](TDiskRegistryDatabase db) + { + TDiskRegistryState::TAllocateDiskResult result{}; + auto error = state.AllocateDisk( + TInstant::Now(), + db, + diskParams, + &result); + UNIT_ASSERT_VALUES_EQUAL(S_OK, error.GetCode()); + }); + + executor.WriteTx( + [&](TDiskRegistryDatabase db) + { + UNIT_ASSERT_SUCCESS( + state.MarkDiskForCleanup(db, diskParams.DiskId)); + UNIT_ASSERT_SUCCESS( + state.DeallocateDisk(db, diskParams.DiskId)); + for (const auto& device: state.GetDirtyDevices()) { + state.MarkDeviceAsClean(Now(), db, device.GetDeviceUUID()); + } + }); + } +} + +#define CREATE_AND_DELETE_NRD_DISK(deviceCount) \ + void CreateDeleteDisk_##deviceCount(benchmark::State& benchmarkState) \ + { \ + TDiskRegistryState::TAllocateDiskParams params{ \ + .DiskId = "disk-1", \ + .PlacementGroupId = {}, \ + .BlockSize = 4_KB, \ + .BlocksCount = 93_GB * (deviceCount) / 4_KB, \ + }; \ + DoCreateDeleteDisk(benchmarkState, params); \ + } \ + BENCHMARK(CreateDeleteDisk_##deviceCount); + +#define CREATE_AND_DELETE_MIRROR_DISK(deviceCount) \ + void CreateDeleteMirrorDisk_##deviceCount( \ + benchmark::State& benchmarkState) \ + { \ + TDiskRegistryState::TAllocateDiskParams params{ \ + .DiskId = "disk-1", \ + .PlacementGroupId = {}, \ + .BlockSize = 4_KB, \ + .BlocksCount = 93_GB * (deviceCount) / 4_KB, \ + .ReplicaCount = 2}; \ + DoCreateDeleteDisk(benchmarkState, params); \ + } \ + BENCHMARK(CreateDeleteMirrorDisk_##deviceCount); + +BENCHMARK(PublishCounters_All); +BENCHMARK(PublishCounters_DisableFullGroups); +CREATE_AND_DELETE_NRD_DISK(1); +CREATE_AND_DELETE_NRD_DISK(10); +CREATE_AND_DELETE_NRD_DISK(100); +CREATE_AND_DELETE_MIRROR_DISK(1); +CREATE_AND_DELETE_MIRROR_DISK(10); +CREATE_AND_DELETE_MIRROR_DISK(100); + +/////////////////////////////////////////////////////////////////////////////// + +} // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/disk_registry/ya.make b/cloud/blockstore/libs/storage/disk_registry/ya.make index fc8d924270c..2428d312d23 100644 --- a/cloud/blockstore/libs/storage/disk_registry/ya.make +++ b/cloud/blockstore/libs/storage/disk_registry/ya.make @@ -93,6 +93,7 @@ RECURSE( ) RECURSE_FOR_TESTS( + benchmark ut ut_allocation ut_cms diff --git a/cloud/blockstore/tests/.gitignore b/cloud/blockstore/tests/.gitignore index 44dd3320f63..7c85016c34d 100644 --- a/cloud/blockstore/tests/.gitignore +++ b/cloud/blockstore/tests/.gitignore @@ -38,6 +38,8 @@ python_client/cloud-blockstore-tests-python_client rdma/image/cloud rdma/image/rootfs.img rdma/rdma-test/cloud-blockstore-tests-rdma-rdma-test +recipes/disk-registry-state/recipe +recipes/disk-registry-state/data/* recipes/endpoint/endpoint-recipe recipes/local-kikimr/local-kikimr-recipe recipes/local-null/local-null-recipe diff --git a/cloud/blockstore/tests/recipes/disk-registry-state/__main__.py b/cloud/blockstore/tests/recipes/disk-registry-state/__main__.py new file mode 100644 index 00000000000..94ba4067e91 --- /dev/null +++ b/cloud/blockstore/tests/recipes/disk-registry-state/__main__.py @@ -0,0 +1,24 @@ +import yatest.common as yatest_common + +from yatest.common import process +from library.python.testing.recipe import declare_recipe + +BACKUP_PATH = "cloud/blockstore/tests/recipes/disk-registry-state/data/backup.json" +GENERATOR_PATH = "cloud/blockstore/tools/testing/disk-registry-state-generator/disk-registry-state-generator" + + +def start(argv): + backup_path = yatest_common.source_path(BACKUP_PATH) + process.execute([ + yatest_common.binary_path(GENERATOR_PATH), + "--backup-path", + backup_path, + ]) + + +def stop(argv): + None + + +if __name__ == "__main__": + declare_recipe(start, stop) diff --git a/cloud/blockstore/tests/recipes/disk-registry-state/recipe.inc b/cloud/blockstore/tests/recipes/disk-registry-state/recipe.inc new file mode 100644 index 00000000000..172bb0d63b7 --- /dev/null +++ b/cloud/blockstore/tests/recipes/disk-registry-state/recipe.inc @@ -0,0 +1,12 @@ +SET(RECIPE_DEPS_LIST + cloud/blockstore/tests/recipes/disk-registry-state + cloud/blockstore/tools/testing/disk-registry-state-generator +) + +DEPENDS(${RECIPE_DEPS_LIST}) + +DATA( + arcadia/cloud/blockstore/tests/recipes/disk-registry-state/data +) + +USE_RECIPE(cloud/blockstore/tests/recipes/disk-registry-state/recipe) diff --git a/cloud/blockstore/tests/recipes/disk-registry-state/ya.make b/cloud/blockstore/tests/recipes/disk-registry-state/ya.make new file mode 100644 index 00000000000..36f8f63a073 --- /dev/null +++ b/cloud/blockstore/tests/recipes/disk-registry-state/ya.make @@ -0,0 +1,13 @@ +PY3_PROGRAM(recipe) + +PY_SRCS( + __main__.py +) + +PEERDIR( + cloud/blockstore/tests/python/lib + cloud/tasks/test/common + library/python/testing/recipe +) + +END() diff --git a/cloud/blockstore/tests/recipes/ya.make b/cloud/blockstore/tests/recipes/ya.make index 32a57e9b099..9ea792e3b00 100644 --- a/cloud/blockstore/tests/recipes/ya.make +++ b/cloud/blockstore/tests/recipes/ya.make @@ -1,4 +1,5 @@ RECURSE( + disk-registry-state endpoint fake-root-kms local-kikimr diff --git a/cloud/blockstore/tools/.gitignore b/cloud/blockstore/tools/.gitignore index 75a62bd7ed6..e64f4dd4aa0 100644 --- a/cloud/blockstore/tools/.gitignore +++ b/cloud/blockstore/tools/.gitignore @@ -48,6 +48,7 @@ testing/chaos-monkey/chaos-monkey testing/chaos-monkey/tests/tests testing/csi-sanity/bin/cloud testing/csi-sanity/bin/csi-sanity +testing/disk-registry-state-generator/disk-registry-state-generator testing/eternal_tests/checkpoint-validator/bin/checkpoint-validator testing/eternal_tests/checkpoint-validator/lib/ut/lib-ut testing/eternal_tests/eternal-load/bin/eternal-load diff --git a/cloud/blockstore/tools/testing/disk-registry-state-generator/main.cpp b/cloud/blockstore/tools/testing/disk-registry-state-generator/main.cpp new file mode 100644 index 00000000000..44d873e7736 --- /dev/null +++ b/cloud/blockstore/tools/testing/disk-registry-state-generator/main.cpp @@ -0,0 +1,390 @@ +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include +#include + +#include + +using namespace NCloud::NBlockStore; +using namespace NCloud::NBlockStore::NStorage; + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +constexpr ui32 NrdBlockSize = 4096; +constexpr ui32 LocalBlockSize = 512; +constexpr size_t NrdDiskInPlacementGroups = 800; + +enum class EDevicePool +{ + Nrd, + V1, + V3, + Chunk29, + Chunk58, +}; + +enum class EPlacementGroupType +{ + Spread, + Partition, +}; + +enum class EVolumeType +{ + V1, + V3, + Nrd, + Mirror3, +}; + +template +struct TEntityInfo +{ + size_t Count = 0; + size_t Min = 0; + size_t Max = 0; + TTag Tag{}; +}; + +constexpr TEntityInfo Racks[]{ + {.Count = 250}, +}; + +constexpr TEntityInfo Hosts[]{ + {.Count = 550, .Min = 15, .Max = 16, .Tag = EDevicePool::Nrd}, + {.Count = 3000, .Min = 31, .Max = 32, .Tag = EDevicePool::Nrd}, + {.Count = 600, .Min = 64, .Max = 64, .Tag = EDevicePool::Nrd}, + {.Count = 300, .Min = 15, .Max = 15, .Tag = EDevicePool::V1}, + {.Count = 500, .Min = 8, .Max = 8, .Tag = EDevicePool::V3}, + {.Count = 200, .Min = 4, .Max = 6, .Tag = EDevicePool::Chunk29}, + {.Count = 25, .Min = 4, .Max = 4, .Tag = EDevicePool::Chunk58}, +}; + +constexpr TEntityInfo PlacementGroups[]{ + {.Count = 2000, .Min = 0, .Max = 0, .Tag = EPlacementGroupType::Spread}, + {.Count = 500, .Min = 3, .Max = 5, .Tag = EPlacementGroupType::Partition}, +}; + +constexpr TEntityInfo Disks[]{ + {.Count = 250, .Min = 1, .Max = 1, .Tag = EVolumeType::V1}, + {.Count = 2500, .Min = 1, .Max = 1, .Tag = EVolumeType::V3}, + {.Count = 3000, .Min = 1, .Max = 1, .Tag = EVolumeType::Nrd}, + {.Count = 2000, .Min = 2, .Max = 3, .Tag = EVolumeType::Nrd}, + {.Count = 1500, .Min = 4, .Max = 11, .Tag = EVolumeType::Nrd}, + {.Count = 400, .Min = 12, .Max = 22, .Tag = EVolumeType::Nrd}, + {.Count = 400, .Min = 23, .Max = 150, .Tag = EVolumeType::Nrd}, + {.Count = 25, .Min = 151, .Max = 800, .Tag = EVolumeType::Nrd}, + {.Count = 750, .Min = 1, .Max = 1, .Tag = EVolumeType::Mirror3}, + {.Count = 350, .Min = 2, .Max = 2, .Tag = EVolumeType::Mirror3}, + {.Count = 150, .Min = 3, .Max = 3, .Tag = EVolumeType::Mirror3}, + {.Count = 300, .Min = 4, .Max = 14, .Tag = EVolumeType::Mirror3}, + {.Count = 150, .Min = 15, .Max = 40, .Tag = EVolumeType::Mirror3}, + {.Count = 100, .Min = 41, .Max = 50, .Tag = EVolumeType::Mirror3}, + {.Count = 20, .Min = 51, .Max = 100, .Tag = EVolumeType::Mirror3}, + {.Count = 5, .Min = 101, .Max = 150, .Tag = EVolumeType::Mirror3}, +}; + +struct TOptions +{ + TString BackupPath = "backup.json"; + ui64 Seed = 42; + + void Parse(int argc, char** argv) + { + NLastGetopt::TOpts opts; + opts.AddHelpOption(); + + opts.AddLongOption("backup-path") + .RequiredArgument() + .StoreResult(&BackupPath); + + opts.AddLongOption("seed") + .OptionalArgument("NUM") + .StoreResult(&Seed); + + NLastGetopt::TOptsParseResultException res(&opts, argc, argv); + } +}; + +template +void Generate( + const std::span>& entities, + std::function generator, + TFastRng64& rand) +{ + size_t i = 0; + for (const auto& entityInfo: entities) { + for (size_t cnt = 0; cnt < entityInfo.Count; ++cnt) { + auto val = + (rand.GenRand() % (entityInfo.Max - entityInfo.Min + 2)) + + entityInfo.Min; + generator(i++, val, entityInfo.Tag); + } + } +} + +auto GenerateAll(ui64 seed) +{ + TMap AllocationUnit; + AllocationUnit[""] = 93_GB; + AllocationUnit["standard-v1"] = 98924756992; + AllocationUnit["standard-v3"] = 394062200832; + AllocationUnit["chunk-2.90TiB"] = 3198924357632; + AllocationUnit["chunk-5.82TiB"] = 6398924554240; + + TFastRng64 rand(seed); + + TVector racks; + auto makeRack = [&](size_t i, size_t val, bool tag) + { + Y_UNUSED(val); + Y_UNUSED(tag); + + racks.push_back(TStringBuilder() << "Rack_" << i); + }; + Generate( + {std::begin(Racks), std::end(Racks)}, + std::function(makeRack), + rand); + + TVector agents; + auto makeHost = [&](size_t i, size_t deviceCount, EDevicePool tag) + { + auto rack = racks[rand.GenRand() % racks.size()]; + + NProto::TAgentConfig agent; + agent.SetNodeId(i); + agent.SetAgentId(TStringBuilder() << "Agent_" << i); + for (size_t j = 0; j < deviceCount; ++j) { + auto* device = agent.MutableDevices()->Add(); + device->SetAgentId(agent.GetAgentId()); + device->SetDeviceName( + TStringBuilder() << "device_" << i << "_" << j); + device->SetDeviceUUID(TStringBuilder() << "uuid_" << i << "_" << j); + device->SetRack(rack); + switch (tag) { + case EDevicePool::Nrd: { + device->SetBlockSize(NrdBlockSize); + device->SetBlocksCount( + AllocationUnit[""] / DefaultBlockSize); + } break; + case EDevicePool::V1: { + device->SetBlockSize(LocalBlockSize); + device->SetBlocksCount( + AllocationUnit["standard-v1"] / LocalBlockSize); + device->SetPoolName("standard-v1"); + device->SetPoolKind( + NProto::EDevicePoolKind::DEVICE_POOL_KIND_LOCAL); + } break; + case EDevicePool::V3: { + device->SetBlockSize(LocalBlockSize); + device->SetBlocksCount( + AllocationUnit["standard-v3"] / LocalBlockSize); + device->SetPoolName("standard-v3"); + device->SetPoolKind( + NProto::EDevicePoolKind::DEVICE_POOL_KIND_LOCAL); + } break; + case EDevicePool::Chunk29: { + device->SetBlockSize(LocalBlockSize); + device->SetBlocksCount( + AllocationUnit["chunk-2.90TiB"] / LocalBlockSize); + device->SetPoolName("chunk-2.90TiB"); + device->SetPoolKind( + NProto::EDevicePoolKind::DEVICE_POOL_KIND_LOCAL); + } break; + case EDevicePool::Chunk58: { + device->SetBlockSize(LocalBlockSize); + device->SetBlocksCount( + AllocationUnit["chunk-5.82TiB"] / LocalBlockSize); + device->SetPoolName("chunk-5.82TiB"); + device->SetPoolKind( + NProto::EDevicePoolKind::DEVICE_POOL_KIND_LOCAL); + } break; + } + } + agents.push_back(std::move(agent)); + }; + Generate( + {std::begin(Hosts), std::end(Hosts)}, + std::function(makeHost), + rand); + + TVector placementGroups; + auto makePlacementGroup = + [&](size_t i, size_t groupCount, EPlacementGroupType tag) + { + NProto::TPlacementGroupConfig config; + switch (tag) { + case EPlacementGroupType::Spread: { + config.SetGroupId(TStringBuilder() << "Spread_" << i); + config.SetPlacementStrategy( + NProto::EPlacementStrategy::PLACEMENT_STRATEGY_SPREAD); + } break; + case EPlacementGroupType::Partition: { + config.SetGroupId(TStringBuilder() << "Partition_" << i); + config.SetPlacementStrategy( + NProto::EPlacementStrategy::PLACEMENT_STRATEGY_PARTITION); + config.SetPlacementPartitionCount(groupCount); + } break; + } + placementGroups.push_back(std::move(config)); + }; + Generate( + {std::begin(PlacementGroups), std::end(PlacementGroups)}, + std::function( + makePlacementGroup), + rand); + + auto monitoring = NCloud::CreateMonitoringServiceStub(); + auto diskRegistryGroup = monitoring->GetCounters() + ->GetSubgroup("counters", "blockstore") + ->GetSubgroup("component", "disk_registry"); + auto state = + NDiskRegistryStateTest::TDiskRegistryStateBuilder() + .AddDevicePoolConfig("", 93_GB, NProto::DEVICE_POOL_KIND_DEFAULT) + .With(diskRegistryGroup) + .WithAgents(std::move(agents)) + .WithPlacementGroups(placementGroups) + .AddDevicePoolConfig( + "standard-v1", + AllocationUnit["standard-v1"], + NProto::DEVICE_POOL_KIND_LOCAL) + .AddDevicePoolConfig( + "standard-v3", + AllocationUnit["standard-v3"], + NProto::DEVICE_POOL_KIND_LOCAL) + .AddDevicePoolConfig( + "chunk-2.90TiB", + AllocationUnit["chunk-2.90TiB"], + NProto::DEVICE_POOL_KIND_LOCAL) + .AddDevicePoolConfig( + "chunk-5.82TiB", + AllocationUnit["chunk-5.82TiB"], + NProto::DEVICE_POOL_KIND_LOCAL) + .Build(); + + TTestExecutor executor; + executor.WriteTx([&](TDiskRegistryDatabase db) mutable + { db.InitSchema(); }); + + // Count probability for placement group + const auto totalNrdCount = Accumulate( + Disks, + size_t{}, + [](size_t acc, const TEntityInfo& diskInfo) + { + return acc + + (diskInfo.Tag == EVolumeType::Nrd ? diskInfo.Count : 0); + }); + const auto placementGroupProbability = + static_cast(NrdDiskInPlacementGroups) / totalNrdCount; + + auto makeDisk = [&](size_t i, size_t deviceCount, EVolumeType tag) + { + TString placementGroupId; + ui32 placementPartitionIndex = 0; + const bool isInPlacementGroup = + rand.GenRand() < + placementGroupProbability * + std::numeric_limits::max(); + if (tag == EVolumeType::Nrd && isInPlacementGroup) { + auto indx = rand.GenRand() % placementGroups.size(); + placementGroupId = placementGroups[indx].GetGroupId(); + if (auto partCount = + placementGroups[indx].GetPlacementPartitionCount()) + { + placementPartitionIndex = rand.GenRand() % partCount; + } + } + + auto mediaKind = NProto::STORAGE_MEDIA_SSD_NONREPLICATED; + TString poolName; + ui32 blockSize = DefaultBlockSize; + switch (tag) { + case EVolumeType::V1: + poolName = "standard-v1"; + mediaKind = NProto::STORAGE_MEDIA_SSD_LOCAL; + blockSize = LocalBlockSize; + break; + case EVolumeType::V3: + poolName = "standard-v3"; + mediaKind = NProto::STORAGE_MEDIA_SSD_LOCAL; + blockSize = LocalBlockSize; + break; + case EVolumeType::Mirror3: + mediaKind = NProto::STORAGE_MEDIA_SSD_MIRROR3; + break; + case EVolumeType::Nrd: + break; + } + + TDiskRegistryState::TAllocateDiskParams diskParams{ + .DiskId = TStringBuilder() << "disk_" << i, + .PlacementGroupId = placementGroupId, + .PlacementPartitionIndex = placementPartitionIndex, + .BlockSize = blockSize, + .BlocksCount = AllocationUnit[poolName] * deviceCount / blockSize, + .ReplicaCount = + static_cast(tag == EVolumeType::Mirror3 ? 2 : 0), + .PoolName = poolName, + .MediaKind = mediaKind, + }; + + executor.WriteTx( + [&](TDiskRegistryDatabase db) + { + TDiskRegistryState::TAllocateDiskResult result{}; + auto error = state.AllocateDisk( + TInstant::Now(), + db, + diskParams, + &result); + }); + }; + Generate( + {std::begin(Disks), std::end(Disks)}, + std::function(makeDisk), + rand); + + { // Output stat. + state.PublishCounters(TInstant::Now()); + for (size_t i = 0; i < state.GetAgents().size(); ++i) { + diskRegistryGroup->RemoveSubgroup( + "agent", + TStringBuilder() << "Agent_" << i); + } + + Cout << "Finish generate state:\n"; + diskRegistryGroup->OutputPlainText(Cout, "\t"); + } + return state; +} + +} // namespace + +int main(int argc, char** argv) +{ + TOptions opts; + opts.Parse(argc, argv); + + auto state = GenerateAll(opts.Seed); + NProto::TDiskRegistryStateBackup backupState = state.BackupState(); + NProto::TBackupDiskRegistryStateResponse backup; + backup.MutableBackup()->Swap(&backupState); + + TProtoStringType str; + google::protobuf::util::MessageToJsonString(backup, &str); + TFileOutput(opts.BackupPath).Write(str.c_str()); + return 0; +} diff --git a/cloud/blockstore/tools/testing/disk-registry-state-generator/ya.make b/cloud/blockstore/tools/testing/disk-registry-state-generator/ya.make new file mode 100644 index 00000000000..9c3923d23f7 --- /dev/null +++ b/cloud/blockstore/tools/testing/disk-registry-state-generator/ya.make @@ -0,0 +1,12 @@ +PROGRAM() + +SRCS( + main.cpp +) + +PEERDIR( + cloud/blockstore/libs/storage/disk_registry + cloud/blockstore/libs/storage/disk_registry/testlib +) + +END() diff --git a/cloud/blockstore/tools/testing/ya.make b/cloud/blockstore/tools/testing/ya.make index 596f387fa5f..26f6171f3e1 100644 --- a/cloud/blockstore/tools/testing/ya.make +++ b/cloud/blockstore/tools/testing/ya.make @@ -2,9 +2,11 @@ RECURSE( bad-guest build-fresh-blob chaos-monkey + disk-registry-state-generator eternal_tests fake-conductor fake-nbs + fake-root-kms fake-vhost-server generate-agents infra-client @@ -14,7 +16,6 @@ RECURSE( pd-metadata-bench plugintest rdma-test - fake-root-kms stable-plugin verify-test )