From ff8bf6c03a6bff6de64f13cb7f2420fd2997b25f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9E=D0=BB=D0=B5=D0=B3?= <150132506+iddqdex@users.noreply.github.com> Date: Fri, 14 Jun 2024 18:56:20 +0300 Subject: [PATCH] add --decimal, use Dt64 as default for init command (#5562) --- .../workload/benchmark_base/workload.cpp | 7 +++++-- ydb/library/workload/benchmark_base/workload.h | 5 +++-- ydb/library/workload/tpch/tpch_schema.sql | 18 +++++++++--------- ydb/tests/functional/clickbench/test.py | 16 +++++++--------- 4 files changed, 24 insertions(+), 22 deletions(-) diff --git a/ydb/library/workload/benchmark_base/workload.cpp b/ydb/library/workload/benchmark_base/workload.cpp index 2b849f5a2b6a..5fa447515f2e 100644 --- a/ydb/library/workload/benchmark_base/workload.cpp +++ b/ydb/library/workload/benchmark_base/workload.cpp @@ -67,6 +67,7 @@ std::string TWorkloadGeneratorBase::GetDDLQueries() const { SubstGlobal(createSql, "{string_type}", Params.GetStringType()); SubstGlobal(createSql, "{date_type}", Params.GetDateType()); SubstGlobal(createSql, "{timestamp_type}", Params.GetTimestampType()); + SubstGlobal(createSql, "{float_type}", Params.GetFloatType()); return createSql.c_str(); } @@ -116,8 +117,10 @@ void TWorkloadBaseParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommand .Optional() .StoreResult(&S3Endpoint); opts.AddLongOption("string", "Use String type in tables instead Utf8 one.").NoArgument().StoreValue(&StringType, "String"); - opts.AddLongOption("date32", "Use Date32 type in tables instead Date one.").NoArgument().StoreValue(&DateType, "Date32"); - opts.AddLongOption("timestamp64", "Use Timestamp64 type in tables instead Timestamp one.").NoArgument().StoreValue(&TimestampType, "Timestamp64"); + opts.AddLongOption("datetime", "Use Date and Timestamp types in tables instead Date32 and Timestamp64 ones.").NoArgument() + .StoreValue(&DateType, "Date").StoreValue(&TimestampType, "Timestamp"); + opts.AddLongOption("decimal", "Use Decimal(22,9) type in tables instead Double one.").NoArgument() + .StoreValue(&FloatType, "Decimal(22,9)"); break; case TWorkloadParams::ECommandType::Root: opts.AddLongOption('p', "path", "Path where benchmark tables are located") diff --git a/ydb/library/workload/benchmark_base/workload.h b/ydb/library/workload/benchmark_base/workload.h index 66e7f2fdaeff..5cf0b77a8c49 100644 --- a/ydb/library/workload/benchmark_base/workload.h +++ b/ydb/library/workload/benchmark_base/workload.h @@ -24,8 +24,9 @@ class TWorkloadBaseParams: public TWorkloadParams { YDB_READONLY_DEF(TString, S3Endpoint); YDB_READONLY_DEF(TString, S3Prefix); YDB_READONLY(TString, StringType, "Utf8"); - YDB_READONLY(TString, DateType, "Date"); - YDB_READONLY(TString, TimestampType, "Timestamp"); + YDB_READONLY(TString, DateType, "Date32"); + YDB_READONLY(TString, TimestampType, "Timestamp64"); + YDB_READONLY(TString, FloatType, "Double"); }; class TWorkloadGeneratorBase : public IWorkloadQueryGenerator { diff --git a/ydb/library/workload/tpch/tpch_schema.sql b/ydb/library/workload/tpch/tpch_schema.sql index 06f42dd0f0f5..434d510f2d23 100644 --- a/ydb/library/workload/tpch/tpch_schema.sql +++ b/ydb/library/workload/tpch/tpch_schema.sql @@ -1,7 +1,7 @@ {createExternal} CREATE {external} TABLE `{path}/customer` ( - c_acctbal Double {notnull}, -- it should be Decimal(12, 2) + c_acctbal {float_type} {notnull}, -- it should be Decimal(12, 2) c_address {string_type} {notnull}, c_comment {string_type} {notnull}, c_custkey Int64 {notnull}, -- Identifier @@ -19,20 +19,20 @@ WITH ({store}"{s3_prefix}/customer/", CREATE {external} TABLE `{path}/lineitem` ( l_comment {string_type} {notnull}, l_commitdate {date_type} {notnull}, - l_discount Double {notnull}, -- it should be Decimal(12, 2) - l_extendedprice Double {notnull}, -- it should be Decimal(12, 2) + l_discount {float_type} {notnull}, -- it should be Decimal(12, 2) + l_extendedprice {float_type} {notnull}, -- it should be Decimal(12, 2) l_linenumber Int32 {notnull}, l_linestatus {string_type} {notnull}, l_orderkey Int64 {notnull}, -- FK to O_ORDERKEY l_partkey Int64 {notnull}, -- FK to P_PARTKEY, first part of the compound FK to (PS_PARTKEY, PS_SUPPKEY) with L_SUPPKEY - l_quantity Double {notnull}, -- it should be Decimal(12, 2) + l_quantity {float_type} {notnull}, -- it should be Decimal(12, 2) l_receiptdate {date_type} {notnull}, l_returnflag {string_type} {notnull}, l_shipdate {date_type} {notnull}, l_shipinstruct {string_type} {notnull}, l_shipmode {string_type} {notnull}, l_suppkey Int64 {notnull}, -- FK to S_SUPPKEY, second part of the compound FK to (PS_PARTKEY, PS_SUPPKEY) with L_PARTKEY - l_tax Double {notnull} -- it should be Decimal(12, 2) + l_tax {float_type} {notnull} -- it should be Decimal(12, 2) {primary_key} (l_orderkey, l_linenumber) ) {partition_by}(l_orderkey) @@ -62,7 +62,7 @@ CREATE {external} TABLE `{path}/orders` ( o_orderpriority {string_type} {notnull}, o_orderstatus {string_type} {notnull}, o_shippriority Int32 {notnull}, - o_totalprice Double {notnull} -- it should be Decimal(12, 2) + o_totalprice {float_type} {notnull} -- it should be Decimal(12, 2) {primary_key} (o_orderkey) ) {partition_by}(o_orderkey) @@ -77,7 +77,7 @@ CREATE {external} TABLE `{path}/part` ( p_mfgr {string_type} {notnull}, p_name {string_type} {notnull}, p_partkey Int64 {notnull}, -- Identifier - p_retailprice Double {notnull}, -- it should be Decimal(12, 2) + p_retailprice {float_type} {notnull}, -- it should be Decimal(12, 2) p_size Int32 {notnull}, p_type {string_type} {notnull} {primary_key}(p_partkey) @@ -92,7 +92,7 @@ CREATE {external} TABLE `{path}/partsupp` ( ps_comment {string_type} {notnull}, ps_partkey Int64 {notnull}, -- FK to P_PARTKEY ps_suppkey Int64 {notnull}, -- FK to S_SUPPKEY - ps_supplycost Double {notnull} -- it should be Decimal(12, 2) + ps_supplycost {float_type} {notnull} -- it should be Decimal(12, 2) {primary_key}(ps_partkey, ps_suppkey) ) {partition_by}(ps_partkey) @@ -112,7 +112,7 @@ WITH ({store}"{s3_prefix}/region/", ); CREATE {external} TABLE `{path}/supplier` ( - s_acctbal Double {notnull}, -- it should be Decimal(12, 2) + s_acctbal {float_type} {notnull}, -- it should be Decimal(12, 2) s_address {string_type} {notnull}, s_comment {string_type} {notnull}, s_name {string_type} {notnull}, diff --git a/ydb/tests/functional/clickbench/test.py b/ydb/tests/functional/clickbench/test.py index 5d8f3eb5148c..7c2a517bd36c 100644 --- a/ydb/tests/functional/clickbench/test.py +++ b/ydb/tests/functional/clickbench/test.py @@ -106,14 +106,13 @@ def save_canonical_data(data, fname): @pytest.mark.parametrize("executer", ["scan", "generic"]) def test_run_benchmark(store, executer): path = "clickbench/benchmark/{}/hits".format(store) - ret = run_cli(["workload", "clickbench", "--path", path, "init", "--store", store]) + ret = run_cli(["workload", "clickbench", "--path", path, "init", "--store", store, "--datetime"]) assert_that(ret.exit_code, is_(0)) ret = run_cli( [ - "import", "file", "csv", "--path", path, - "--input-file", - yatest.common.source_path("ydb/tests/functional/clickbench/data/hits.csv") + "workload", "clickbench", "--path", path, "import", "files", + "--input", yatest.common.source_path("ydb/tests/functional/clickbench/data/hits.csv") ] ) assert_that(ret.exit_code, is_(0)) @@ -127,13 +126,12 @@ def test_run_benchmark(store, executer): @pytest.mark.parametrize("store", ["row", "column"]) def test_run_determentistic(store): path = "clickbench/determentistic/{}/hits".format(store) - ret = run_cli(["workload", "clickbench", "--path", path, "init", "--store", store]) + ret = run_cli(["workload", "clickbench", "--path", path, "init", "--store", store, "--datetime"]) assert_that(ret.exit_code, is_(0)) ret = run_cli( [ - "import", "file", "csv", "--path", path, - "--input-file", - yatest.common.source_path("ydb/tests/functional/clickbench/data/hits.csv") + "workload", "clickbench", "--path", path, "import", "files", + "--input", yatest.common.source_path("ydb/tests/functional/clickbench/data/hits.csv") ] ) assert_that(ret.exit_code, is_(0)) @@ -158,7 +156,7 @@ def test_run_determentistic(store): @pytest.mark.parametrize("store", ["row", "column"]) def test_plans(store): ret = run_cli( - ["workload", "clickbench", "--path", "clickbench/plans/{}/hits".format(store), "init", "--store", store] + ["workload", "clickbench", "--path", "clickbench/plans/{}/hits".format(store), "init", "--store", store, "--datetime"] ) assert_that(ret.exit_code, is_(0))