Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add use-intermediate-write-buffer tag if scrubbing found a mismatch #2836

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions cloud/blockstore/config/storage.proto
Original file line number Diff line number Diff line change
Expand Up @@ -1083,4 +1083,8 @@ message TStorageServiceConfig
// percentage, then the rejection of such agents does not occur - we assume
// a connectivity failure in the cluster.
optional double DiskRegistryInitialAgentRejectionThreshold = 396;

// When enabled, tag "use-intermediate-write-buffer" will be added
// after scrubbing finds a mismatch
optional bool AutomaticallyEnableBufferCopyingAfterChecksumMismatch = 397;
}
26 changes: 26 additions & 0 deletions cloud/blockstore/libs/storage/api/service.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ namespace NCloud::NBlockStore::NStorage {
xxx(ChangeVolumeBinding, __VA_ARGS__) \
xxx(GetVolumeStats, __VA_ARGS__) \
xxx(RunVolumesLivenessCheck, __VA_ARGS__) \
xxx(AddTags, __VA_ARGS__) \
// BLOCKSTORE_SERVICE_REQUESTS

////////////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -160,6 +161,28 @@ struct TEvService
{}
};

//
// AddTags
//

struct TAddTagsRequest
{
const TString DiskId;
const TVector<TString> Tags;

TAddTagsRequest() = default;

TAddTagsRequest(
TString diskId,
TVector<TString> tags)
: DiskId(std::move(diskId))
, Tags(std::move(tags))
{}
};

struct TAddTagsResponse
{};

//
// VolumeMountStateChanged
//
Expand Down Expand Up @@ -317,6 +340,9 @@ struct TEvService
EvQueryAgentsInfoRequest = EvBegin + 89,
EvQueryAgentsInfoResponse = EvBegin + 90,

EvAddTagsRequest = EvBegin + 91,
EvAddTagsResponse = EvBegin + 92,

EvEnd
};

Expand Down
2 changes: 2 additions & 0 deletions cloud/blockstore/libs/storage/core/config.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -521,6 +521,8 @@ TDuration MSeconds(ui32 value)
xxx(EncryptionAtRestForDiskRegistryBasedDisksEnabled, bool, false )\
xxx(DisableFullPlacementGroupCountCalculation, bool, false )\
xxx(DiskRegistryInitialAgentRejectionThreshold, double, 50 )\
\
xxx(AutomaticallyEnableBufferCopyingAfterChecksumMismatch, bool, false )\
// BLOCKSTORE_STORAGE_CONFIG_RW

#define BLOCKSTORE_STORAGE_CONFIG(xxx) \
Expand Down
2 changes: 2 additions & 0 deletions cloud/blockstore/libs/storage/core/config.h
Original file line number Diff line number Diff line change
Expand Up @@ -622,6 +622,8 @@ class TStorageConfig

[[nodiscard]] bool GetDisableFullPlacementGroupCountCalculation() const;
[[nodiscard]] double GetDiskRegistryInitialAgentRejectionThreshold() const;

bool GetAutomaticallyEnableBufferCopyingAfterChecksumMismatch() const;
};

ui64 GetAllocationUnit(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
#include <cloud/blockstore/libs/storage/core/probes.h>
#include <cloud/blockstore/libs/storage/core/proto_helpers.h>
#include <cloud/blockstore/libs/storage/core/unimplemented.h>
#include <cloud/blockstore/libs/storage/volume/volume_state.h>
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

не надо создавать кольцевую зависимость

volume зависит от partition_nonrepl
в обратную сторону зависимость делать не надо


#include <contrib/ydb/core/base/appdata.h>

Expand Down Expand Up @@ -204,6 +205,11 @@ void TMirrorPartitionActor::CompareChecksums(const TActorContext& ctx)
DiskId.c_str(),
DescribeRange(GetScrubbingRange()).c_str());

if (Config->GetAutomaticallyEnableBufferCopyingAfterChecksumMismatch())
{
AddTagForBufferCopying(ctx);
}

for (size_t i = 0; i < checksums.size(); i++) {
LOG_ERROR(
ctx,
Expand Down Expand Up @@ -271,6 +277,25 @@ void TMirrorPartitionActor::StartResyncRange(
BlockDigestGenerator);
}

void TMirrorPartitionActor::AddTagForBufferCopying(
const NActors::TActorContext& ctx)
{
auto requestInfo = CreateRequestInfo(
SelfId(),
0, // cookie
MakeIntrusive<TCallContext>()
);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

на предыдущую строку


TVector<TString> tags(1);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

это вектор длины 1
в следующей строке добавляется второй элемент
сделай просто TVector tags({IntermediateWriteBufferTagName});

tags.emplace_back(IntermediateWriteBufferTagName);
auto request = std::make_unique<TEvService::TEvAddTagsRequest>(
DiskId,
std::move(tags)
);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

на предыдущую строку


ctx.Send(MakeStorageServiceId(), std::move(request));
}

void TMirrorPartitionActor::ReplyAndDie(const TActorContext& ctx)
{
NCloud::Reply(ctx, *Poisoner, std::make_unique<TEvents::TEvPoisonTaken>());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ class TMirrorPartitionActor final
const NActors::TActorContext& ctx,
ui64 scrubbingRangeId);
void StartResyncRange(const NActors::TActorContext& ctx);
void AddTagForBufferCopying(const NActors::TActorContext& ctx);

private:
STFUNC(StateWork);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,15 @@ struct TTestEnv
)
);

Runtime.AddLocalService(
MakeStorageServiceId(),
TActorSetupCmd(
new TStorageServiceMock(),
TMailboxType::Simple,
0
)
);

NKikimr::SetupTabletServices(Runtime);
}

Expand Down Expand Up @@ -1176,7 +1185,20 @@ Y_UNIT_TEST_SUITE(TMirrorPartitionTest)
TDynamicCountersPtr critEventsCounters = new TDynamicCounters();
InitCriticalEventsCounter(critEventsCounters);

TTestEnv env(runtime);
NProto::TStorageServiceConfig config;
config.SetAutomaticallyEnableBufferCopyingAfterChecksumMismatch(true);
TTestEnv env(runtime, config);

bool tagEnabled = false;
runtime.SetEventFilter([&] (auto& runtime, auto& event) {
Y_UNUSED(runtime);
if (event->GetTypeRewrite() == TEvService::EvAddTagsRequest)
{
tagEnabled = true;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

проверь, что отправляется правильный тег

}

return false;
});

const auto range1 = TBlockRange64::WithLength(0, 2);
env.WriteMirror(range1, 'A');
Expand All @@ -1197,6 +1219,7 @@ Y_UNIT_TEST_SUITE(TMirrorPartitionTest)

UNIT_ASSERT_VALUES_EQUAL(2, mirroredDiskMinorityChecksumMismatch->Val());
UNIT_ASSERT_VALUES_EQUAL(2, counters.Simple.ChecksumMismatches.Value);
UNIT_ASSERT(tagEnabled);

const auto range3 = TBlockRange64::WithLength(1025, 50);
env.WriteMirror(range3, 'A');
Expand Down
42 changes: 42 additions & 0 deletions cloud/blockstore/libs/storage/partition_nonrepl/ut_env.h
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,48 @@ class TDummyActor final

////////////////////////////////////////////////////////////////////////////////

class TStorageServiceMock final
: public NActors::TActor<TStorageServiceMock>
{
public:
TStorageServiceMock()
: TActor(&TThis::StateWork)
{
}

private:
STFUNC(StateWork)
{
switch (ev->GetTypeRewrite()) {
HFunc(NActors::TEvents::TEvPoisonPill, HandlePoisonPill);

HFunc(TEvService::TEvAddTagsRequest, HandleAddTagsRequest);

default:
Y_ABORT("Unexpected event %x", ev->GetTypeRewrite());
}
}

void HandlePoisonPill(
const NActors::TEvents::TEvPoisonPill::TPtr& ev,
const NActors::TActorContext& ctx)
{
Y_UNUSED(ev);

Die(ctx);
}

void HandleAddTagsRequest(
const TEvService::TEvAddTagsRequest::TPtr& ev,
const NActors::TActorContext& ctx)
{
Y_UNUSED(ev);
Y_UNUSED(ctx);
}
};

////////////////////////////////////////////////////////////////////////////////

class TPartitionClient
{
private:
Expand Down
4 changes: 4 additions & 0 deletions cloud/blockstore/libs/storage/service/service_actor.h
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,10 @@ class TServiceActor final
const TEvService::TEvRunVolumesLivenessCheckResponse::TPtr& ev,
const NActors::TActorContext& ctx);

void HandleAddTagsRequest(
const TEvService::TEvAddTagsRequest::TPtr& ev,
const NActors::TActorContext& ctx);

bool HandleRequests(STFUNC_SIG);

BLOCKSTORE_STORAGE_SERVICE(BLOCKSTORE_IMPLEMENT_REQUEST, TEvService)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,35 @@ STFUNC(TModifyTagsActionActor::StateWaitReady)

////////////////////////////////////////////////////////////////////////////////

void TServiceActor::HandleAddTagsRequest(
const TEvService::TEvAddTagsRequest::TPtr& ev,
const NActors::TActorContext& ctx)
{
auto* msg = ev->Get();

auto requestInfo = CreateRequestInfo(
SelfId(),
0, // cookie
MakeIntrusive<TCallContext>()
);

NPrivateProto::TModifyTagsRequest modifyTagsRequest;
modifyTagsRequest.SetDiskId(msg->DiskId);
for (const auto& tag : msg->Tags) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

пробел перед : не нужен

modifyTagsRequest.AddTagsToAdd(tag);
}
TString input;
google::protobuf::util::MessageToJsonString(modifyTagsRequest, &input);

NCloud::Register(
ctx,
std::make_unique<TModifyTagsActionActor>(
std::move(requestInfo),
std::move(input)));
}

////////////////////////////////////////////////////////////////////////////////

TResultOrError<IActorPtr> TServiceActor::CreateModifyTagsActionActor(
TRequestInfoPtr requestInfo,
TString input)
Expand Down
2 changes: 1 addition & 1 deletion cloud/blockstore/libs/storage/volume/volume_state.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ void TVolumeState::Reset()
TDuration::TryParse(value, MaxTimedOutDeviceStateDuration);
} else if (tag == "use-fastpath") {
UseFastPath = true;
} else if (tag == "use-intermediate-write-buffer") {
} else if (tag == IntermediateWriteBufferTagName) {
UseIntermediateWriteBuffer = true;
}
}
Expand Down
5 changes: 5 additions & 0 deletions cloud/blockstore/libs/storage/volume/volume_state.h
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,11 @@ using TMigrations = google::protobuf::RepeatedPtrField<NProto::TDeviceMigration>

////////////////////////////////////////////////////////////////////////////////

constexpr TStringBuf IntermediateWriteBufferTagName =
"use-intermediate-write-buffer";

////////////////////////////////////////////////////////////////////////////////

struct THistoryLogKey
{
TInstant Timestamp;
Expand Down
Loading