From 14fbdb09f11e05a625c58c5701aa920222109c49 Mon Sep 17 00:00:00 2001 From: Kirill Pleshivtsev Date: Tue, 14 Jan 2025 20:32:49 +0700 Subject: [PATCH] Make generation like real cluster --- .../storage/disk_registry/benchmark/ya.make | 1 - .../disk_registry_state_benchmark.cpp | 6 +- .../recipes/disk-registry-state/__main__.py | 4 +- .../disk-registry-state-generator/main.cpp | 241 ++++++++++++++++-- 4 files changed, 218 insertions(+), 34 deletions(-) diff --git a/cloud/blockstore/libs/storage/disk_registry/benchmark/ya.make b/cloud/blockstore/libs/storage/disk_registry/benchmark/ya.make index b62d4ef3a4..3c732ec963 100644 --- a/cloud/blockstore/libs/storage/disk_registry/benchmark/ya.make +++ b/cloud/blockstore/libs/storage/disk_registry/benchmark/ya.make @@ -16,5 +16,4 @@ PEERDIR( cloud/blockstore/libs/storage/disk_registry/testlib ) - END() diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_state_benchmark.cpp b/cloud/blockstore/libs/storage/disk_registry/disk_registry_state_benchmark.cpp index a57eac24f9..49c3940774 100644 --- a/cloud/blockstore/libs/storage/disk_registry/disk_registry_state_benchmark.cpp +++ b/cloud/blockstore/libs/storage/disk_registry/disk_registry_state_benchmark.cpp @@ -125,8 +125,10 @@ void DoCreateDeleteDisk( executor.WriteTx( [&](TDiskRegistryDatabase db) { - UNIT_ASSERT_SUCCESS(state.MarkDiskForCleanup(db, "disk-1")); - UNIT_ASSERT_SUCCESS(state.DeallocateDisk(db, "disk-1")); + UNIT_ASSERT_SUCCESS( + state.MarkDiskForCleanup(db, diskParams.DiskId)); + UNIT_ASSERT_SUCCESS( + state.DeallocateDisk(db, diskParams.DiskId)); for (const auto& device: state.GetDirtyDevices()) { state.MarkDeviceAsClean(Now(), db, device.GetDeviceUUID()); } diff --git a/cloud/blockstore/tests/recipes/disk-registry-state/__main__.py b/cloud/blockstore/tests/recipes/disk-registry-state/__main__.py index 7b2b2bbe91..94ba4067e9 100644 --- a/cloud/blockstore/tests/recipes/disk-registry-state/__main__.py +++ b/cloud/blockstore/tests/recipes/disk-registry-state/__main__.py @@ -1,9 +1,7 @@ -import json -import os import yatest.common as yatest_common from yatest.common import process -from library.python.testing.recipe import declare_recipe, set_env +from library.python.testing.recipe import declare_recipe BACKUP_PATH = "cloud/blockstore/tests/recipes/disk-registry-state/data/backup.json" GENERATOR_PATH = "cloud/blockstore/tools/testing/disk-registry-state-generator/disk-registry-state-generator" diff --git a/cloud/blockstore/tools/testing/disk-registry-state-generator/main.cpp b/cloud/blockstore/tools/testing/disk-registry-state-generator/main.cpp index 3ed57bfe69..44d873e773 100644 --- a/cloud/blockstore/tools/testing/disk-registry-state-generator/main.cpp +++ b/cloud/blockstore/tools/testing/disk-registry-state-generator/main.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -20,17 +21,29 @@ namespace { //////////////////////////////////////////////////////////////////////////////// -constexpr ui32 DefaultBlockSize = 4096; +constexpr ui32 NrdBlockSize = 4096; +constexpr ui32 LocalBlockSize = 512; +constexpr size_t NrdDiskInPlacementGroups = 800; enum class EDevicePool { - Local, Nrd, + V1, + V3, + Chunk29, + Chunk58, +}; + +enum class EPlacementGroupType +{ + Spread, + Partition, }; enum class EVolumeType { - Local, + V1, + V3, Nrd, Mirror3, }; @@ -49,21 +62,43 @@ constexpr TEntityInfo Racks[]{ }; constexpr TEntityInfo Hosts[]{ - {.Count = 1000, .Min = 15, .Max = 16, .Tag = EDevicePool::Nrd}, - {.Count = 1000, .Min = 31, .Max = 32, .Tag = EDevicePool::Nrd}, - {.Count = 1000, .Min = 64, .Max = 64, .Tag = EDevicePool::Nrd}, - {.Count = 1000, .Min = 80, .Max = 80, .Tag = EDevicePool::Nrd}, - {.Count = 1000, .Min = 8, .Max = 8, .Tag = EDevicePool::Local}, + {.Count = 550, .Min = 15, .Max = 16, .Tag = EDevicePool::Nrd}, + {.Count = 3000, .Min = 31, .Max = 32, .Tag = EDevicePool::Nrd}, + {.Count = 600, .Min = 64, .Max = 64, .Tag = EDevicePool::Nrd}, + {.Count = 300, .Min = 15, .Max = 15, .Tag = EDevicePool::V1}, + {.Count = 500, .Min = 8, .Max = 8, .Tag = EDevicePool::V3}, + {.Count = 200, .Min = 4, .Max = 6, .Tag = EDevicePool::Chunk29}, + {.Count = 25, .Min = 4, .Max = 4, .Tag = EDevicePool::Chunk58}, +}; + +constexpr TEntityInfo PlacementGroups[]{ + {.Count = 2000, .Min = 0, .Max = 0, .Tag = EPlacementGroupType::Spread}, + {.Count = 500, .Min = 3, .Max = 5, .Tag = EPlacementGroupType::Partition}, }; constexpr TEntityInfo Disks[]{ - {.Count = 1000, .Min = 1, .Max = 3, .Tag = EVolumeType::Nrd}, - {.Count = 1000, .Min = 1, .Max = 3, .Tag = EVolumeType::Mirror3}, + {.Count = 250, .Min = 1, .Max = 1, .Tag = EVolumeType::V1}, + {.Count = 2500, .Min = 1, .Max = 1, .Tag = EVolumeType::V3}, + {.Count = 3000, .Min = 1, .Max = 1, .Tag = EVolumeType::Nrd}, + {.Count = 2000, .Min = 2, .Max = 3, .Tag = EVolumeType::Nrd}, + {.Count = 1500, .Min = 4, .Max = 11, .Tag = EVolumeType::Nrd}, + {.Count = 400, .Min = 12, .Max = 22, .Tag = EVolumeType::Nrd}, + {.Count = 400, .Min = 23, .Max = 150, .Tag = EVolumeType::Nrd}, + {.Count = 25, .Min = 151, .Max = 800, .Tag = EVolumeType::Nrd}, + {.Count = 750, .Min = 1, .Max = 1, .Tag = EVolumeType::Mirror3}, + {.Count = 350, .Min = 2, .Max = 2, .Tag = EVolumeType::Mirror3}, + {.Count = 150, .Min = 3, .Max = 3, .Tag = EVolumeType::Mirror3}, + {.Count = 300, .Min = 4, .Max = 14, .Tag = EVolumeType::Mirror3}, + {.Count = 150, .Min = 15, .Max = 40, .Tag = EVolumeType::Mirror3}, + {.Count = 100, .Min = 41, .Max = 50, .Tag = EVolumeType::Mirror3}, + {.Count = 20, .Min = 51, .Max = 100, .Tag = EVolumeType::Mirror3}, + {.Count = 5, .Min = 101, .Max = 150, .Tag = EVolumeType::Mirror3}, }; struct TOptions { - TString BackupPath; + TString BackupPath = "backup.json"; + ui64 Seed = 42; void Parse(int argc, char** argv) { @@ -74,6 +109,10 @@ struct TOptions .RequiredArgument() .StoreResult(&BackupPath); + opts.AddLongOption("seed") + .OptionalArgument("NUM") + .StoreResult(&Seed); + NLastGetopt::TOptsParseResultException res(&opts, argc, argv); } }; @@ -95,9 +134,16 @@ void Generate( } } -auto GenerateAll() +auto GenerateAll(ui64 seed) { - TFastRng64 Rand(GetCycleCount()); + TMap AllocationUnit; + AllocationUnit[""] = 93_GB; + AllocationUnit["standard-v1"] = 98924756992; + AllocationUnit["standard-v3"] = 394062200832; + AllocationUnit["chunk-2.90TiB"] = 3198924357632; + AllocationUnit["chunk-5.82TiB"] = 6398924554240; + + TFastRng64 rand(seed); TVector racks; auto makeRack = [&](size_t i, size_t val, bool tag) @@ -110,13 +156,12 @@ auto GenerateAll() Generate( {std::begin(Racks), std::end(Racks)}, std::function(makeRack), - Rand); + rand); TVector agents; auto makeHost = [&](size_t i, size_t deviceCount, EDevicePool tag) { - Y_UNUSED(tag); - auto rack = racks[Rand.GenRand() % racks.size()]; + auto rack = racks[rand.GenRand() % racks.size()]; NProto::TAgentConfig agent; agent.SetNodeId(i); @@ -127,16 +172,79 @@ auto GenerateAll() device->SetDeviceName( TStringBuilder() << "device_" << i << "_" << j); device->SetDeviceUUID(TStringBuilder() << "uuid_" << i << "_" << j); - device->SetBlockSize(DefaultBlockSize); - device->SetBlocksCount(24379392); device->SetRack(rack); + switch (tag) { + case EDevicePool::Nrd: { + device->SetBlockSize(NrdBlockSize); + device->SetBlocksCount( + AllocationUnit[""] / DefaultBlockSize); + } break; + case EDevicePool::V1: { + device->SetBlockSize(LocalBlockSize); + device->SetBlocksCount( + AllocationUnit["standard-v1"] / LocalBlockSize); + device->SetPoolName("standard-v1"); + device->SetPoolKind( + NProto::EDevicePoolKind::DEVICE_POOL_KIND_LOCAL); + } break; + case EDevicePool::V3: { + device->SetBlockSize(LocalBlockSize); + device->SetBlocksCount( + AllocationUnit["standard-v3"] / LocalBlockSize); + device->SetPoolName("standard-v3"); + device->SetPoolKind( + NProto::EDevicePoolKind::DEVICE_POOL_KIND_LOCAL); + } break; + case EDevicePool::Chunk29: { + device->SetBlockSize(LocalBlockSize); + device->SetBlocksCount( + AllocationUnit["chunk-2.90TiB"] / LocalBlockSize); + device->SetPoolName("chunk-2.90TiB"); + device->SetPoolKind( + NProto::EDevicePoolKind::DEVICE_POOL_KIND_LOCAL); + } break; + case EDevicePool::Chunk58: { + device->SetBlockSize(LocalBlockSize); + device->SetBlocksCount( + AllocationUnit["chunk-5.82TiB"] / LocalBlockSize); + device->SetPoolName("chunk-5.82TiB"); + device->SetPoolKind( + NProto::EDevicePoolKind::DEVICE_POOL_KIND_LOCAL); + } break; + } } agents.push_back(std::move(agent)); }; Generate( {std::begin(Hosts), std::end(Hosts)}, std::function(makeHost), - Rand); + rand); + + TVector placementGroups; + auto makePlacementGroup = + [&](size_t i, size_t groupCount, EPlacementGroupType tag) + { + NProto::TPlacementGroupConfig config; + switch (tag) { + case EPlacementGroupType::Spread: { + config.SetGroupId(TStringBuilder() << "Spread_" << i); + config.SetPlacementStrategy( + NProto::EPlacementStrategy::PLACEMENT_STRATEGY_SPREAD); + } break; + case EPlacementGroupType::Partition: { + config.SetGroupId(TStringBuilder() << "Partition_" << i); + config.SetPlacementStrategy( + NProto::EPlacementStrategy::PLACEMENT_STRATEGY_PARTITION); + config.SetPlacementPartitionCount(groupCount); + } break; + } + placementGroups.push_back(std::move(config)); + }; + Generate( + {std::begin(PlacementGroups), std::end(PlacementGroups)}, + std::function( + makePlacementGroup), + rand); auto monitoring = NCloud::CreateMonitoringServiceStub(); auto diskRegistryGroup = monitoring->GetCounters() @@ -147,25 +255,91 @@ auto GenerateAll() .AddDevicePoolConfig("", 93_GB, NProto::DEVICE_POOL_KIND_DEFAULT) .With(diskRegistryGroup) .WithAgents(std::move(agents)) + .WithPlacementGroups(placementGroups) + .AddDevicePoolConfig( + "standard-v1", + AllocationUnit["standard-v1"], + NProto::DEVICE_POOL_KIND_LOCAL) + .AddDevicePoolConfig( + "standard-v3", + AllocationUnit["standard-v3"], + NProto::DEVICE_POOL_KIND_LOCAL) + .AddDevicePoolConfig( + "chunk-2.90TiB", + AllocationUnit["chunk-2.90TiB"], + NProto::DEVICE_POOL_KIND_LOCAL) + .AddDevicePoolConfig( + "chunk-5.82TiB", + AllocationUnit["chunk-5.82TiB"], + NProto::DEVICE_POOL_KIND_LOCAL) .Build(); TTestExecutor executor; executor.WriteTx([&](TDiskRegistryDatabase db) mutable { db.InitSchema(); }); + // Count probability for placement group + const auto totalNrdCount = Accumulate( + Disks, + size_t{}, + [](size_t acc, const TEntityInfo& diskInfo) + { + return acc + + (diskInfo.Tag == EVolumeType::Nrd ? diskInfo.Count : 0); + }); + const auto placementGroupProbability = + static_cast(NrdDiskInPlacementGroups) / totalNrdCount; + auto makeDisk = [&](size_t i, size_t deviceCount, EVolumeType tag) { + TString placementGroupId; + ui32 placementPartitionIndex = 0; + const bool isInPlacementGroup = + rand.GenRand() < + placementGroupProbability * + std::numeric_limits::max(); + if (tag == EVolumeType::Nrd && isInPlacementGroup) { + auto indx = rand.GenRand() % placementGroups.size(); + placementGroupId = placementGroups[indx].GetGroupId(); + if (auto partCount = + placementGroups[indx].GetPlacementPartitionCount()) + { + placementPartitionIndex = rand.GenRand() % partCount; + } + } + + auto mediaKind = NProto::STORAGE_MEDIA_SSD_NONREPLICATED; + TString poolName; + ui32 blockSize = DefaultBlockSize; + switch (tag) { + case EVolumeType::V1: + poolName = "standard-v1"; + mediaKind = NProto::STORAGE_MEDIA_SSD_LOCAL; + blockSize = LocalBlockSize; + break; + case EVolumeType::V3: + poolName = "standard-v3"; + mediaKind = NProto::STORAGE_MEDIA_SSD_LOCAL; + blockSize = LocalBlockSize; + break; + case EVolumeType::Mirror3: + mediaKind = NProto::STORAGE_MEDIA_SSD_MIRROR3; + break; + case EVolumeType::Nrd: + break; + } + TDiskRegistryState::TAllocateDiskParams diskParams{ .DiskId = TStringBuilder() << "disk_" << i, - .PlacementGroupId = {}, - .BlockSize = 4_KB, - .BlocksCount = 93_GB * (deviceCount) / 4_KB, + .PlacementGroupId = placementGroupId, + .PlacementPartitionIndex = placementPartitionIndex, + .BlockSize = blockSize, + .BlocksCount = AllocationUnit[poolName] * deviceCount / blockSize, .ReplicaCount = static_cast(tag == EVolumeType::Mirror3 ? 2 : 0), - .MediaKind = - (tag == EVolumeType::Mirror3 - ? NProto::STORAGE_MEDIA_SSD_MIRROR3 - : NProto::STORAGE_MEDIA_SSD_NONREPLICATED)}; + .PoolName = poolName, + .MediaKind = mediaKind, + }; executor.WriteTx( [&](TDiskRegistryDatabase db) @@ -181,8 +355,19 @@ auto GenerateAll() Generate( {std::begin(Disks), std::end(Disks)}, std::function(makeDisk), - Rand); + rand); + + { // Output stat. + state.PublishCounters(TInstant::Now()); + for (size_t i = 0; i < state.GetAgents().size(); ++i) { + diskRegistryGroup->RemoveSubgroup( + "agent", + TStringBuilder() << "Agent_" << i); + } + Cout << "Finish generate state:\n"; + diskRegistryGroup->OutputPlainText(Cout, "\t"); + } return state; } @@ -193,7 +378,7 @@ int main(int argc, char** argv) TOptions opts; opts.Parse(argc, argv); - auto state = GenerateAll(); + auto state = GenerateAll(opts.Seed); NProto::TDiskRegistryStateBackup backupState = state.BackupState(); NProto::TBackupDiskRegistryStateResponse backup; backup.MutableBackup()->Swap(&backupState);