diff --git a/ydb/library/yaml_config/yaml_config_parser.cpp b/ydb/library/yaml_config/yaml_config_parser.cpp index 11e1bbeb7079..c1e354881a77 100644 --- a/ydb/library/yaml_config/yaml_config_parser.cpp +++ b/ydb/library/yaml_config/yaml_config_parser.cpp @@ -1488,6 +1488,15 @@ namespace NKikimr::NYaml { TTransformContext ctx; NKikimrConfig::TEphemeralInputFields ephemeralConfig; + if (json.Has("metadata")) { + ValidateMetadata(json["metadata"]); + + Y_ENSURE_BT(json.Has("config") && json["config"].IsMap(), + "'config' must be an object when 'metadata' is present"); + + jsonNode = json["config"]; + } + if (transform) { ExtractExtraFields(jsonNode, ctx); @@ -1509,11 +1518,17 @@ namespace NKikimr::NYaml { NJson::TJsonValue jsonNode = Yaml2Json(yamlNode, true); NKikimrConfig::TAppConfig config; + Parse(jsonNode, GetJsonToProtoConfig(), config, transform); return config; } + void ValidateMetadata(const NJson::TJsonValue& metadata) { + Y_ENSURE_BT(metadata.Has("cluster") && metadata["cluster"].IsString(), "Metadata must contain a string 'cluster' field"); + Y_ENSURE_BT(metadata.Has("version") && metadata["version"].IsUInteger(), "Metadata must contain an unsigned int 'version' field"); + } + } // NKikimr::NYaml template <> diff --git a/ydb/library/yaml_config/yaml_config_parser.h b/ydb/library/yaml_config/yaml_config_parser.h index 1343ece220c9..af988aff3aa7 100644 --- a/ydb/library/yaml_config/yaml_config_parser.h +++ b/ydb/library/yaml_config/yaml_config_parser.h @@ -67,4 +67,6 @@ namespace NKikimr::NYaml { void Parse(const NJson::TJsonValue& json, NProtobufJson::TJson2ProtoConfig convertConfig, NKikimrConfig::TAppConfig& config, bool transform, bool relaxed = false); NKikimrConfig::TAppConfig Parse(const TString& data, bool transform = true); + void ValidateMetadata(const NJson::TJsonValue& metadata); + } // namespace NKikimr::NYaml diff --git a/ydb/tests/functional/config/test_config_with_metadata.py b/ydb/tests/functional/config/test_config_with_metadata.py new file mode 100644 index 000000000000..e04225f55476 --- /dev/null +++ b/ydb/tests/functional/config/test_config_with_metadata.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +import logging +import time +from hamcrest import assert_that + +from ydb.tests.library.common.types import Erasure +import ydb.tests.library.common.cms as cms +from ydb.tests.library.clients.kikimr_http_client import SwaggerClient +from ydb.tests.library.harness.util import LogLevels +from ydb.tests.library.harness.kikimr_runner import KiKiMR +from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator +from ydb.tests.library.kv.helpers import create_kv_tablets_and_wait_for_start +from ydb.public.api.protos.ydb_status_codes_pb2 import StatusIds + + +logger = logging.getLogger(__name__) + + +def value_for(key, tablet_id): + return "Value: ".format( + key=key, tablet_id=tablet_id) + + +class AbstractKiKiMRTest(object): + erasure = None + metadata_section = None + + @classmethod + def setup_class(cls): + nodes_count = 8 if cls.erasure == Erasure.BLOCK_4_2 else 9 + configurator = KikimrConfigGenerator(cls.erasure, + nodes=nodes_count, + use_in_memory_pdisks=False, + additional_log_configs={'CMS': LogLevels.DEBUG}, + metadata_section=cls.metadata_section, + ) + cls.cluster = KiKiMR(configurator=configurator) + cls.cluster.start() + + time.sleep(120) + cms.request_increase_ratio_limit(cls.cluster.client) + host = cls.cluster.nodes[1].host + mon_port = cls.cluster.nodes[1].mon_port + cls.swagger_client = SwaggerClient(host, mon_port) + + @classmethod + def teardown_class(cls): + cls.cluster.stop() + + +class TestKiKiMRWithMetadata(AbstractKiKiMRTest): + erasure = Erasure.BLOCK_4_2 + metadata_section = { + 'cluster': 'test_cluster', + 'version': 1 + } + + def test_cluster_is_operational_with_metadata(self): + table_path = '/Root/mydb/mytable_with_metadata' + number_of_tablets = 5 + tablet_ids = create_kv_tablets_and_wait_for_start( + self.cluster.client, + self.cluster.kv_client, + self.swagger_client, + number_of_tablets, + table_path, + timeout_seconds=120 + ) + + for partition_id, tablet_id in enumerate(tablet_ids): + resp = self.cluster.kv_client.kv_write(table_path, partition_id, "key", value_for("key", tablet_id)) + assert_that(resp.operation.status == StatusIds.SUCCESS) + + resp = self.cluster.kv_client.kv_read(table_path, partition_id, "key") + assert_that(resp.operation.status == StatusIds.SUCCESS) + + +class TestKiKiMRWithoutMetadata(AbstractKiKiMRTest): + erasure = Erasure.BLOCK_4_2 + + def test_cluster_is_operational_without_metadata(self): + table_path = '/Root/mydb/mytable_without_metadata' + number_of_tablets = 5 + tablet_ids = create_kv_tablets_and_wait_for_start( + self.cluster.client, + self.cluster.kv_client, + self.swagger_client, + number_of_tablets, + table_path, + timeout_seconds=120 + ) + + for partition_id, tablet_id in enumerate(tablet_ids): + resp = self.cluster.kv_client.kv_write(table_path, partition_id, "key", value_for("key", tablet_id)) + assert_that(resp.operation.status == StatusIds.SUCCESS) + + resp = self.cluster.kv_client.kv_read(table_path, partition_id, "key") + assert_that(resp.operation.status == StatusIds.SUCCESS) + + +class TestConfigWithMetadataBlock(TestKiKiMRWithMetadata): + erasure = Erasure.BLOCK_4_2 + + +class TestConfigWithoutMetadataBlock(TestKiKiMRWithoutMetadata): + erasure = Erasure.BLOCK_4_2 + + +class TestConfigWithMetadataMirrorMax(TestKiKiMRWithMetadata): + erasure = Erasure.MIRROR_3_DC + + +class TestConfigWithoutMetadataMirror(TestKiKiMRWithoutMetadata): + erasure = Erasure.MIRROR_3_DC diff --git a/ydb/tests/functional/config/ya.make b/ydb/tests/functional/config/ya.make new file mode 100644 index 000000000000..2e3d969998d5 --- /dev/null +++ b/ydb/tests/functional/config/ya.make @@ -0,0 +1,36 @@ +PY3TEST() + +TEST_SRCS( + test_config_with_metadata.py +) + +SPLIT_FACTOR(10) + +IF (SANITIZER_TYPE) + REQUIREMENTS(ram:16 cpu:4) +ENDIF() + +IF (SANITIZER_TYPE == "thread") + TIMEOUT(1800) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + + +ENV(YDB_DRIVER_BINARY="ydb/apps/ydbd/ydbd") +DEPENDS( + ydb/apps/ydbd +) + +PEERDIR( + ydb/tests/library + ydb/tests/library/clients +) + +FORK_SUBTESTS() +FORK_TEST_FILES() + +END() diff --git a/ydb/tests/functional/ya.make b/ydb/tests/functional/ya.make index 1d20092f929f..9332e328f739 100644 --- a/ydb/tests/functional/ya.make +++ b/ydb/tests/functional/ya.make @@ -8,6 +8,7 @@ RECURSE( clickbench cms compatibility + config dynumber encryption hive diff --git a/ydb/tests/library/harness/kikimr_config.py b/ydb/tests/library/harness/kikimr_config.py index 862698891a7b..5feaa7309ecf 100644 --- a/ydb/tests/library/harness/kikimr_config.py +++ b/ydb/tests/library/harness/kikimr_config.py @@ -156,6 +156,7 @@ def __init__( default_user_sid=None, pg_compatible_expirement=False, generic_connector_config=None, # typing.Optional[TGenericConnectorConfig] + metadata_section=None, ): if extra_feature_flags is None: extra_feature_flags = [] @@ -430,6 +431,13 @@ def __init__( self.yaml_config["feature_flags"]["enable_external_data_sources"] = True self.yaml_config["feature_flags"]["enable_script_execution_operations"] = True + self.full_config = dict() + if metadata_section: + self.full_config["metadata"] = metadata_section + self.full_config["config"] = self.yaml_config + else: + self.full_config = self.yaml_config + @property def pdisks_info(self): return self._pdisks_info @@ -526,7 +534,7 @@ def write_tls_data(self): def write_proto_configs(self, configs_path): self.write_tls_data() with open(os.path.join(configs_path, "config.yaml"), "w") as writer: - writer.write(yaml.safe_dump(self.yaml_config)) + writer.write(yaml.safe_dump(self.full_config)) def clone_grpc_as_ext_endpoint(self, port, endpoint_id=None): cur_grpc_config = copy.deepcopy(self.yaml_config['grpc_config'])