Skip to content

Commit

Permalink
Make generation like real cluster
Browse files Browse the repository at this point in the history
  • Loading branch information
@drbasic
drbasic committed Jan 15, 2025
1 parent 2fbdc12 commit 14fbdb0
Show file tree
Hide file tree
Showing 4 changed files with 218 additions and 34 deletions.
1 change: 0 additions & 1 deletion cloud/blockstore/libs/storage/disk_registry/benchmark/ya.make
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,4 @@ PEERDIR(
cloud/blockstore/libs/storage/disk_registry/testlib
)


END()
6 changes: 4 additions & 2 deletions cloud/blockstore/libs/storage/disk_registry/disk_registry_state_benchmark.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,8 +125,10 @@ void DoCreateDeleteDisk(
executor.WriteTx(
[&](TDiskRegistryDatabase db)
{
UNIT_ASSERT_SUCCESS(state.MarkDiskForCleanup(db, "disk-1"));
UNIT_ASSERT_SUCCESS(state.DeallocateDisk(db, "disk-1"));
UNIT_ASSERT_SUCCESS(
state.MarkDiskForCleanup(db, diskParams.DiskId));
UNIT_ASSERT_SUCCESS(
state.DeallocateDisk(db, diskParams.DiskId));
for (const auto& device: state.GetDirtyDevices()) {
state.MarkDeviceAsClean(Now(), db, device.GetDeviceUUID());
}
Expand Down
4 changes: 1 addition & 3 deletions cloud/blockstore/tests/recipes/disk-registry-state/__main__.py
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
import json
import os
import yatest.common as yatest_common

from yatest.common import process
from library.python.testing.recipe import declare_recipe, set_env
from library.python.testing.recipe import declare_recipe

BACKUP_PATH = "cloud/blockstore/tests/recipes/disk-registry-state/data/backup.json"
GENERATOR_PATH = "cloud/blockstore/tools/testing/disk-registry-state-generator/disk-registry-state-generator"
Expand Down
241 changes: 213 additions & 28 deletions cloud/blockstore/tools/testing/disk-registry-state-generator/main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include <contrib/libs/protobuf/src/google/protobuf/util/json_util.h>

#include <library/cpp/getopt/small/last_getopt.h>
#include <library/cpp/monlib/dynamic_counters/encode.h>

#include <util/random/fast.h>
#include <util/stream/file.h>
Expand All @@ -20,17 +21,29 @@ namespace {

////////////////////////////////////////////////////////////////////////////////

constexpr ui32 DefaultBlockSize = 4096;
constexpr ui32 NrdBlockSize = 4096;
constexpr ui32 LocalBlockSize = 512;
constexpr size_t NrdDiskInPlacementGroups = 800;

enum class EDevicePool
{
Local,
Nrd,
V1,
V3,
Chunk29,
Chunk58,
};

enum class EPlacementGroupType
{
Spread,
Partition,
};

enum class EVolumeType
{
Local,
V1,
V3,
Nrd,
Mirror3,
};
Expand All @@ -49,21 +62,43 @@ constexpr TEntityInfo<bool> Racks[]{
};

constexpr TEntityInfo<EDevicePool> Hosts[]{
{.Count = 1000, .Min = 15, .Max = 16, .Tag = EDevicePool::Nrd},
{.Count = 1000, .Min = 31, .Max = 32, .Tag = EDevicePool::Nrd},
{.Count = 1000, .Min = 64, .Max = 64, .Tag = EDevicePool::Nrd},
{.Count = 1000, .Min = 80, .Max = 80, .Tag = EDevicePool::Nrd},
{.Count = 1000, .Min = 8, .Max = 8, .Tag = EDevicePool::Local},
{.Count = 550, .Min = 15, .Max = 16, .Tag = EDevicePool::Nrd},
{.Count = 3000, .Min = 31, .Max = 32, .Tag = EDevicePool::Nrd},
{.Count = 600, .Min = 64, .Max = 64, .Tag = EDevicePool::Nrd},
{.Count = 300, .Min = 15, .Max = 15, .Tag = EDevicePool::V1},
{.Count = 500, .Min = 8, .Max = 8, .Tag = EDevicePool::V3},
{.Count = 200, .Min = 4, .Max = 6, .Tag = EDevicePool::Chunk29},
{.Count = 25, .Min = 4, .Max = 4, .Tag = EDevicePool::Chunk58},
};

constexpr TEntityInfo<EPlacementGroupType> PlacementGroups[]{
{.Count = 2000, .Min = 0, .Max = 0, .Tag = EPlacementGroupType::Spread},
{.Count = 500, .Min = 3, .Max = 5, .Tag = EPlacementGroupType::Partition},
};

constexpr TEntityInfo<EVolumeType> Disks[]{
{.Count = 1000, .Min = 1, .Max = 3, .Tag = EVolumeType::Nrd},
{.Count = 1000, .Min = 1, .Max = 3, .Tag = EVolumeType::Mirror3},
{.Count = 250, .Min = 1, .Max = 1, .Tag = EVolumeType::V1},
{.Count = 2500, .Min = 1, .Max = 1, .Tag = EVolumeType::V3},
{.Count = 3000, .Min = 1, .Max = 1, .Tag = EVolumeType::Nrd},
{.Count = 2000, .Min = 2, .Max = 3, .Tag = EVolumeType::Nrd},
{.Count = 1500, .Min = 4, .Max = 11, .Tag = EVolumeType::Nrd},
{.Count = 400, .Min = 12, .Max = 22, .Tag = EVolumeType::Nrd},
{.Count = 400, .Min = 23, .Max = 150, .Tag = EVolumeType::Nrd},
{.Count = 25, .Min = 151, .Max = 800, .Tag = EVolumeType::Nrd},
{.Count = 750, .Min = 1, .Max = 1, .Tag = EVolumeType::Mirror3},
{.Count = 350, .Min = 2, .Max = 2, .Tag = EVolumeType::Mirror3},
{.Count = 150, .Min = 3, .Max = 3, .Tag = EVolumeType::Mirror3},
{.Count = 300, .Min = 4, .Max = 14, .Tag = EVolumeType::Mirror3},
{.Count = 150, .Min = 15, .Max = 40, .Tag = EVolumeType::Mirror3},
{.Count = 100, .Min = 41, .Max = 50, .Tag = EVolumeType::Mirror3},
{.Count = 20, .Min = 51, .Max = 100, .Tag = EVolumeType::Mirror3},
{.Count = 5, .Min = 101, .Max = 150, .Tag = EVolumeType::Mirror3},
};

struct TOptions
{
TString BackupPath;
TString BackupPath = "backup.json";
ui64 Seed = 42;

void Parse(int argc, char** argv)
{
Expand All @@ -74,6 +109,10 @@ struct TOptions
.RequiredArgument()
.StoreResult(&BackupPath);

opts.AddLongOption("seed")
.OptionalArgument("NUM")
.StoreResult(&Seed);

NLastGetopt::TOptsParseResultException res(&opts, argc, argv);
}
};
Expand All @@ -95,9 +134,16 @@ void Generate(
}
}

auto GenerateAll()
auto GenerateAll(ui64 seed)
{
TFastRng64 Rand(GetCycleCount());
TMap<TString, ui64> AllocationUnit;
AllocationUnit[""] = 93_GB;
AllocationUnit["standard-v1"] = 98924756992;
AllocationUnit["standard-v3"] = 394062200832;
AllocationUnit["chunk-2.90TiB"] = 3198924357632;
AllocationUnit["chunk-5.82TiB"] = 6398924554240;

TFastRng64 rand(seed);

TVector<TString> racks;
auto makeRack = [&](size_t i, size_t val, bool tag)
Expand All @@ -110,13 +156,12 @@ auto GenerateAll()
Generate(
{std::begin(Racks), std::end(Racks)},
std::function<void(size_t i, size_t val, bool tag)>(makeRack),
Rand);
rand);

TVector<NProto::TAgentConfig> agents;
auto makeHost = [&](size_t i, size_t deviceCount, EDevicePool tag)
{
Y_UNUSED(tag);
auto rack = racks[Rand.GenRand() % racks.size()];
auto rack = racks[rand.GenRand() % racks.size()];

NProto::TAgentConfig agent;
agent.SetNodeId(i);
Expand All @@ -127,16 +172,79 @@ auto GenerateAll()
device->SetDeviceName(
TStringBuilder() << "device_" << i << "_" << j);
device->SetDeviceUUID(TStringBuilder() << "uuid_" << i << "_" << j);
device->SetBlockSize(DefaultBlockSize);
device->SetBlocksCount(24379392);
device->SetRack(rack);
switch (tag) {
case EDevicePool::Nrd: {
device->SetBlockSize(NrdBlockSize);
device->SetBlocksCount(
AllocationUnit[""] / DefaultBlockSize);
} break;
case EDevicePool::V1: {
device->SetBlockSize(LocalBlockSize);
device->SetBlocksCount(
AllocationUnit["standard-v1"] / LocalBlockSize);
device->SetPoolName("standard-v1");
device->SetPoolKind(
NProto::EDevicePoolKind::DEVICE_POOL_KIND_LOCAL);
} break;
case EDevicePool::V3: {
device->SetBlockSize(LocalBlockSize);
device->SetBlocksCount(
AllocationUnit["standard-v3"] / LocalBlockSize);
device->SetPoolName("standard-v3");
device->SetPoolKind(
NProto::EDevicePoolKind::DEVICE_POOL_KIND_LOCAL);
} break;
case EDevicePool::Chunk29: {
device->SetBlockSize(LocalBlockSize);
device->SetBlocksCount(
AllocationUnit["chunk-2.90TiB"] / LocalBlockSize);
device->SetPoolName("chunk-2.90TiB");
device->SetPoolKind(
NProto::EDevicePoolKind::DEVICE_POOL_KIND_LOCAL);
} break;
case EDevicePool::Chunk58: {
device->SetBlockSize(LocalBlockSize);
device->SetBlocksCount(
AllocationUnit["chunk-5.82TiB"] / LocalBlockSize);
device->SetPoolName("chunk-5.82TiB");
device->SetPoolKind(
NProto::EDevicePoolKind::DEVICE_POOL_KIND_LOCAL);
} break;
}
}
agents.push_back(std::move(agent));
};
Generate(
{std::begin(Hosts), std::end(Hosts)},
std::function<void(size_t i, size_t val, EDevicePool tag)>(makeHost),
Rand);
rand);

TVector<NProto::TPlacementGroupConfig> placementGroups;
auto makePlacementGroup =
[&](size_t i, size_t groupCount, EPlacementGroupType tag)
{
NProto::TPlacementGroupConfig config;
switch (tag) {
case EPlacementGroupType::Spread: {
config.SetGroupId(TStringBuilder() << "Spread_" << i);
config.SetPlacementStrategy(
NProto::EPlacementStrategy::PLACEMENT_STRATEGY_SPREAD);
} break;
case EPlacementGroupType::Partition: {
config.SetGroupId(TStringBuilder() << "Partition_" << i);
config.SetPlacementStrategy(
NProto::EPlacementStrategy::PLACEMENT_STRATEGY_PARTITION);
config.SetPlacementPartitionCount(groupCount);
} break;
}
placementGroups.push_back(std::move(config));
};
Generate(
{std::begin(PlacementGroups), std::end(PlacementGroups)},
std::function<void(size_t, size_t, EPlacementGroupType)>(
makePlacementGroup),
rand);

auto monitoring = NCloud::CreateMonitoringServiceStub();
auto diskRegistryGroup = monitoring->GetCounters()
Expand All @@ -147,25 +255,91 @@ auto GenerateAll()
.AddDevicePoolConfig("", 93_GB, NProto::DEVICE_POOL_KIND_DEFAULT)
.With(diskRegistryGroup)
.WithAgents(std::move(agents))
.WithPlacementGroups(placementGroups)
.AddDevicePoolConfig(
"standard-v1",
AllocationUnit["standard-v1"],
NProto::DEVICE_POOL_KIND_LOCAL)
.AddDevicePoolConfig(
"standard-v3",
AllocationUnit["standard-v3"],
NProto::DEVICE_POOL_KIND_LOCAL)
.AddDevicePoolConfig(
"chunk-2.90TiB",
AllocationUnit["chunk-2.90TiB"],
NProto::DEVICE_POOL_KIND_LOCAL)
.AddDevicePoolConfig(
"chunk-5.82TiB",
AllocationUnit["chunk-5.82TiB"],
NProto::DEVICE_POOL_KIND_LOCAL)
.Build();

TTestExecutor executor;
executor.WriteTx([&](TDiskRegistryDatabase db) mutable
{ db.InitSchema(); });

// Count probability for placement group
const auto totalNrdCount = Accumulate(
Disks,
size_t{},
[](size_t acc, const TEntityInfo<EVolumeType>& diskInfo)
{
return acc +
(diskInfo.Tag == EVolumeType::Nrd ? diskInfo.Count : 0);
});
const auto placementGroupProbability =
static_cast<double>(NrdDiskInPlacementGroups) / totalNrdCount;

auto makeDisk = [&](size_t i, size_t deviceCount, EVolumeType tag)
{
TString placementGroupId;
ui32 placementPartitionIndex = 0;
const bool isInPlacementGroup =
rand.GenRand() <
placementGroupProbability *
std::numeric_limits<decltype(rand.GenRand())>::max();
if (tag == EVolumeType::Nrd && isInPlacementGroup) {
auto indx = rand.GenRand() % placementGroups.size();
placementGroupId = placementGroups[indx].GetGroupId();
if (auto partCount =
placementGroups[indx].GetPlacementPartitionCount())
{
placementPartitionIndex = rand.GenRand() % partCount;
}
}

auto mediaKind = NProto::STORAGE_MEDIA_SSD_NONREPLICATED;
TString poolName;
ui32 blockSize = DefaultBlockSize;
switch (tag) {
case EVolumeType::V1:
poolName = "standard-v1";
mediaKind = NProto::STORAGE_MEDIA_SSD_LOCAL;
blockSize = LocalBlockSize;
break;
case EVolumeType::V3:
poolName = "standard-v3";
mediaKind = NProto::STORAGE_MEDIA_SSD_LOCAL;
blockSize = LocalBlockSize;
break;
case EVolumeType::Mirror3:
mediaKind = NProto::STORAGE_MEDIA_SSD_MIRROR3;
break;
case EVolumeType::Nrd:
break;
}

TDiskRegistryState::TAllocateDiskParams diskParams{
.DiskId = TStringBuilder() << "disk_" << i,
.PlacementGroupId = {},
.BlockSize = 4_KB,
.BlocksCount = 93_GB * (deviceCount) / 4_KB,
.PlacementGroupId = placementGroupId,
.PlacementPartitionIndex = placementPartitionIndex,
.BlockSize = blockSize,
.BlocksCount = AllocationUnit[poolName] * deviceCount / blockSize,
.ReplicaCount =
static_cast<ui32>(tag == EVolumeType::Mirror3 ? 2 : 0),
.MediaKind =
(tag == EVolumeType::Mirror3
? NProto::STORAGE_MEDIA_SSD_MIRROR3
: NProto::STORAGE_MEDIA_SSD_NONREPLICATED)};
.PoolName = poolName,
.MediaKind = mediaKind,
};

executor.WriteTx(
[&](TDiskRegistryDatabase db)
Expand All @@ -181,8 +355,19 @@ auto GenerateAll()
Generate(
{std::begin(Disks), std::end(Disks)},
std::function<void(size_t i, size_t val, EVolumeType tag)>(makeDisk),
Rand);
rand);

{ // Output stat.
state.PublishCounters(TInstant::Now());
for (size_t i = 0; i < state.GetAgents().size(); ++i) {
diskRegistryGroup->RemoveSubgroup(
"agent",
TStringBuilder() << "Agent_" << i);
}

Cout << "Finish generate state:\n";
diskRegistryGroup->OutputPlainText(Cout, "\t");
}
return state;
}

Expand All @@ -193,7 +378,7 @@ int main(int argc, char** argv)
TOptions opts;
opts.Parse(argc, argv);

auto state = GenerateAll();
auto state = GenerateAll(opts.Seed);
NProto::TDiskRegistryStateBackup backupState = state.BackupState();
NProto::TBackupDiskRegistryStateResponse backup;
backup.MutableBackup()->Swap(&backupState);
Expand Down

0 comments on commit 14fbdb0

Please sign in to comment.