diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql index 4da2e5f44..5464246d5 100755 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql @@ -362,30 +362,39 @@ CREATE TABLE test_udt ( employee_name VARCHAR(100), home_address address_type, some_field enum_test, - home_address1 non_public.address_type1 + home_address1 non_public.address_type1, + scalar_column TEXT CHECK (scalar_column IS JSON SCALAR) ); CREATE TABLE test_arr_enum ( id int, arr text[], - arr_enum enum_test[] + arr_enum enum_test[], + object_column TEXT CHECK (object_column IS JSON OBJECT) ); CREATE TABLE public.locations ( id integer NOT NULL, name character varying(100), - geom geometry(Point,4326) + geom geometry(Point,4326), + array_column TEXT CHECK (array_column IS JSON ARRAY) ); CREATE TABLE public.xml_data_example ( id SERIAL PRIMARY KEY, name VARCHAR(255), description XML DEFAULT xmlparse(document 'Default Product100.00Electronics'), - created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP + created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, + unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS) ); CREATE TABLE image (title text, raster lo); +-- IS JSON Predicate +CREATE TABLE public.json_data ( + id SERIAL PRIMARY KEY, + data_column TEXT NOT NULL CHECK (data_column IS JSON) +); CREATE TABLE employees (id INT PRIMARY KEY, salary INT); -- create table with multirange data types diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index 0bfd66b0e..0fc61bbf6 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -30,6 +30,56 @@ "GH": "", "MinimumVersionsFixedIn": null }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "public.json_data", + "Reason": "Json Type Predicate", + "SqlStatement": "CREATE TABLE public.json_data (\n id SERIAL PRIMARY KEY,\n data_column TEXT NOT NULL CHECK (data_column IS JSON)\n);", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "test_arr_enum", + "Reason": "Json Type Predicate", + "SqlStatement": "CREATE TABLE test_arr_enum (\n\tid int,\n\tarr text[],\n\tarr_enum enum_test[],\n object_column TEXT CHECK (object_column IS JSON OBJECT)\n);", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "test_udt", + "Reason": "Json Type Predicate", + "SqlStatement": "CREATE TABLE test_udt (\n\temployee_id SERIAL PRIMARY KEY,\n\temployee_name VARCHAR(100),\n\thome_address address_type,\n\tsome_field enum_test,\n\thome_address1 non_public.address_type1,\n scalar_column TEXT CHECK (scalar_column IS JSON SCALAR)\n);", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "public.xml_data_example", + "Reason": "Json Type Predicate", + "SqlStatement": " CREATE TABLE public.xml_data_example (\n id SERIAL PRIMARY KEY,\n name VARCHAR(255),\n description XML DEFAULT xmlparse(document '\u003cproduct\u003e\u003cname\u003eDefault Product\u003c/name\u003e\u003cprice\u003e100.00\u003c/price\u003e\u003ccategory\u003eElectronics\u003c/category\u003e\u003c/product\u003e'),\n created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,\n unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS)\n);", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "public.locations", + "Reason": "Json Type Predicate", + "SqlStatement": "CREATE TABLE public.locations (\n id integer NOT NULL,\n name character varying(100),\n geom geometry(Point,4326),\n array_column TEXT CHECK (array_column IS JSON ARRAY)\n );", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null + }, { "IssueType": "unsupported_features", "ObjectType": "VIEW", @@ -55,7 +105,7 @@ "ObjectType": "TABLE", "ObjectName": "public.xml_data_example", "Reason": "XML Functions", - "SqlStatement": " CREATE TABLE public.xml_data_example (\n id SERIAL PRIMARY KEY,\n name VARCHAR(255),\n description XML DEFAULT xmlparse(document '\u003cproduct\u003e\u003cname\u003eDefault Product\u003c/name\u003e\u003cprice\u003e100.00\u003c/price\u003e\u003ccategory\u003eElectronics\u003c/category\u003e\u003c/product\u003e'),\n created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP\n);", + "SqlStatement": " CREATE TABLE public.xml_data_example (\n id SERIAL PRIMARY KEY,\n name VARCHAR(255),\n description XML DEFAULT xmlparse(document '\u003cproduct\u003e\u003cname\u003eDefault Product\u003c/name\u003e\u003cprice\u003e100.00\u003c/price\u003e\u003ccategory\u003eElectronics\u003c/category\u003e\u003c/product\u003e'),\n created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,\n unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS)\n);", "Suggestion": "", "GH": "", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", @@ -66,7 +116,7 @@ "ObjectType": "TABLE", "ObjectName": "public.xml_data_example", "Reason": "Unsupported datatype - xml on column - description", - "SqlStatement": " CREATE TABLE public.xml_data_example (\n id SERIAL PRIMARY KEY,\n name VARCHAR(255),\n description XML DEFAULT xmlparse(document '\u003cproduct\u003e\u003cname\u003eDefault Product\u003c/name\u003e\u003cprice\u003e100.00\u003c/price\u003e\u003ccategory\u003eElectronics\u003c/category\u003e\u003c/product\u003e'),\n created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP\n);", + "SqlStatement": " CREATE TABLE public.xml_data_example (\n id SERIAL PRIMARY KEY,\n name VARCHAR(255),\n description XML DEFAULT xmlparse(document '\u003cproduct\u003e\u003cname\u003eDefault Product\u003c/name\u003e\u003cprice\u003e100.00\u003c/price\u003e\u003ccategory\u003eElectronics\u003c/category\u003e\u003c/product\u003e'),\n created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,\n unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS)\n);", "Suggestion": "Data ingestion is not supported for this type in YugabyteDB so handle this type in different way. Refer link for more details.", "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported", @@ -275,7 +325,7 @@ "ObjectType": "TABLE", "ObjectName": "test_udt", "Reason": "Unsupported datatype for Live migration with fall-forward/fallback - address_type on column - home_address", - "SqlStatement": "CREATE TABLE test_udt (\n\temployee_id SERIAL PRIMARY KEY,\n\temployee_name VARCHAR(100),\n\thome_address address_type,\n\tsome_field enum_test,\n\thome_address1 non_public.address_type1\n);", + "SqlStatement": "CREATE TABLE test_udt (\n\temployee_id SERIAL PRIMARY KEY,\n\temployee_name VARCHAR(100),\n\thome_address address_type,\n\tsome_field enum_test,\n\thome_address1 non_public.address_type1,\n scalar_column TEXT CHECK (scalar_column IS JSON SCALAR)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", @@ -286,7 +336,7 @@ "ObjectType": "TABLE", "ObjectName": "test_arr_enum", "Reason": "Unsupported datatype for Live migration with fall-forward/fallback - enum_test[] on column - arr_enum", - "SqlStatement": "CREATE TABLE test_arr_enum (\n\tid int,\n\tarr text[],\n\tarr_enum enum_test[]\n);", + "SqlStatement": "CREATE TABLE test_arr_enum (\n\tid int,\n\tarr text[],\n\tarr_enum enum_test[],\n object_column TEXT CHECK (object_column IS JSON OBJECT)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", @@ -297,7 +347,7 @@ "ObjectType": "TABLE", "ObjectName": "test_udt", "Reason": "Unsupported datatype for Live migration with fall-forward/fallback - non_public.address_type1 on column - home_address1", - "SqlStatement": "CREATE TABLE test_udt (\n\temployee_id SERIAL PRIMARY KEY,\n\temployee_name VARCHAR(100),\n\thome_address address_type,\n\tsome_field enum_test,\n\thome_address1 non_public.address_type1\n);", + "SqlStatement": "CREATE TABLE test_udt (\n\temployee_id SERIAL PRIMARY KEY,\n\temployee_name VARCHAR(100),\n\thome_address address_type,\n\tsome_field enum_test,\n\thome_address1 non_public.address_type1,\n scalar_column TEXT CHECK (scalar_column IS JSON SCALAR)\n);", "Suggestion": "", "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", @@ -1212,7 +1262,7 @@ "ObjectType": "TABLE", "ObjectName": "public.locations", "Reason": "Unsupported datatype - geometry on column - geom", - "SqlStatement": "CREATE TABLE public.locations (\n id integer NOT NULL,\n name character varying(100),\n geom geometry(Point,4326)\n );", + "SqlStatement": "CREATE TABLE public.locations (\n id integer NOT NULL,\n name character varying(100),\n geom geometry(Point,4326),\n array_column TEXT CHECK (array_column IS JSON ARRAY)\n );", "Suggestion": "", "GH": "https://github.com/yugabyte/yugabyte-db/issues/11323", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb", diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index 5e9353c7a..8cc96548f 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -26,9 +26,9 @@ }, { "ObjectType": "TABLE", - "TotalCount": 58, - "InvalidCount": 49, - "ObjectNames": "employees, image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table" }, + "TotalCount": 59, + "InvalidCount": 50, + "ObjectNames": "public.json_data, employees, image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table" }, { "ObjectType": "INDEX", diff --git a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json index d02458c00..276d6f01e 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json @@ -25,15 +25,15 @@ }, { "ObjectType": "TABLE", - "TotalCount": 2, - "InvalidCount": 0, - "ObjectNames": "analytics.metrics, sales.orders" + "TotalCount": 4, + "InvalidCount": 1, + "ObjectNames": "sales.json_data, analytics.metrics, sales.events, sales.orders" }, { "ObjectType": "VIEW", - "TotalCount": 1, - "InvalidCount": 1, - "ObjectNames": "sales.employ_depart_view" + "TotalCount": 3, + "InvalidCount": 3, + "ObjectNames": "sales.event_analysis_view, sales.event_analysis_view2, sales.employ_depart_view" } ] }, @@ -41,9 +41,11 @@ "SizingRecommendation": { "ColocatedTables": [ "sales.orders", - "analytics.metrics" + "analytics.metrics", + "sales.events", + "sales.json_data" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 2 objects (2 tables/materialized views and 0 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 4 objects (4 tables/materialized views and 0 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": null, "NumNodes": 3, "VCPUsPerInstance": 4, @@ -55,18 +57,43 @@ }, "FailureReasoning": "" }, - "UnsupportedDataTypes": null, + "UnsupportedDataTypes": [ + { + "SchemaName": "sales", + "TableName": "event_analysis_view", + "ColumnName": "all_event_ranges", + "DataType": "datemultirange" + } + ], "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ { "FeatureName": "Aggregate Functions", "Objects": [ + { + "ObjectName": "sales.event_analysis_view", + "SqlStatement": "CREATE VIEW sales.event_analysis_view AS\n SELECT range_agg(event_range) AS all_event_ranges\n FROM sales.events;" + }, + { + "ObjectName": "sales.event_analysis_view2", + "SqlStatement": "CREATE VIEW sales.event_analysis_view2 AS\n SELECT range_intersect_agg(event_range) AS overlapping_range\n FROM sales.events;" + }, { "ObjectName": "sales.employ_depart_view", "SqlStatement": "CREATE VIEW sales.employ_depart_view AS\n SELECT any_value(name) AS any_employee\n FROM public.employees;" } ], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Json Type Predicate", + "Objects": [ + { + "ObjectName": "sales.json_data", + "SqlStatement": "CREATE TABLE sales.json_data (\n id integer NOT NULL,\n array_column text,\n unique_keys_column text,\n CONSTRAINT json_data_array_column_check CHECK ((array_column IS JSON ARRAY)),\n CONSTRAINT json_data_unique_keys_column_check CHECK ((unique_keys_column IS JSON WITH UNIQUE KEYS))\n);" + } + ], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", @@ -85,6 +112,34 @@ "ParentTableName": null, "SizeInBytes": 8192 }, + { + "SchemaName": "sales", + "ObjectName": "events", + "RowCount": 3, + "ColumnCount": 2, + "Reads": 6, + "Writes": 3, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 8192 + }, + { + "SchemaName": "sales", + "ObjectName": "json_data", + "RowCount": 0, + "ColumnCount": 3, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, { "SchemaName": "analytics", "ObjectName": "metrics", @@ -109,11 +164,29 @@ "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", "MinimumVersionsFixedIn": null }, + { + "ConstructTypeName": "Aggregate Functions", + "Query": "SELECT range_intersect_agg(event_range) AS intersection_of_ranges\nFROM sales.events", + "DocsLink": "", + "MinimumVersionsFixedIn": null + }, + { + "ConstructTypeName": "Aggregate Functions", + "Query": "SELECT range_agg(event_range) AS union_of_ranges\nFROM sales.events", + "DocsLink": "", + "MinimumVersionsFixedIn": null + }, { "ConstructTypeName": "Aggregate Functions", "Query": "SELECT\n any_value(name) AS any_employee\n FROM employees", "DocsLink": "", "MinimumVersionsFixedIn": null + }, + { + "ConstructTypeName": "Json Type Predicate", + "Query": "SELECT * \nFROM sales.json_data\nWHERE array_column IS JSON ARRAY", + "DocsLink": "", + "MinimumVersionsFixedIn": null } ], "UnsupportedPlPgSqlObjects": null diff --git a/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql b/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql index 2b42d3334..b4fe35874 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql +++ b/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql @@ -45,4 +45,44 @@ INSERT INTO analytics.metrics (metric_name, metric_value) VALUES ('ConversionRat create view sales.employ_depart_view AS SELECT any_value(name) AS any_employee - FROM employees; \ No newline at end of file + FROM employees; + +CREATE TABLE sales.events ( + id int PRIMARY KEY, + event_range daterange +); + +-- Insert some ranges +INSERT INTO sales.events (id, event_range) VALUES + (1,'[2024-01-01, 2024-01-10]'::daterange), + (2,'[2024-01-05, 2024-01-15]'::daterange), + (3,'[2024-01-20, 2024-01-25]'::daterange); + +CREATE VIEW sales.event_analysis_view AS +SELECT + range_agg(event_range) AS all_event_ranges +FROM + sales.events; + +CREATE VIEW sales.event_analysis_view2 AS +SELECT + range_intersect_agg(event_range) AS overlapping_range +FROM + sales.events; + +-- PG 16 and above feature +CREATE TABLE sales.json_data ( + id int PRIMARY KEY, + array_column TEXT CHECK (array_column IS JSON ARRAY), + unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS) +); + +INSERT INTO public.json_data ( + id, data_column, object_column, array_column, scalar_column, unique_keys_column +) VALUES ( + 1, '{"key": "value"}', + 2, '{"name": "John", "age": 30}', + 3, '[1, 2, 3, 4]', + 4, '"hello"', + 5, '{"uniqueKey1": "value1", "uniqueKey2": "value2"}' +); \ No newline at end of file diff --git a/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql b/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql index 1840c3624..43a68b8e3 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql +++ b/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql @@ -25,4 +25,16 @@ WHERE metric_value > 0.02; -- Aggregate functions UQC NOT REPORTING as it need PG16 upgarde in pipeline from PG15 SELECT any_value(name) AS any_employee - FROM employees; \ No newline at end of file + FROM employees; + +--PG15 +SELECT range_agg(event_range) AS union_of_ranges +FROM sales.events; + +SELECT range_intersect_agg(event_range) AS intersection_of_ranges +FROM sales.events; + +-- -- PG 16 and above feature +SELECT * +FROM sales.json_data +WHERE array_column IS JSON ARRAY; \ No newline at end of file diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 1c70f78b8..d007fa217 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -1043,6 +1043,7 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_CONSTRUCTOR_FUNCTION_NAME, "", queryissue.JSON_CONSTRUCTOR_FUNCTION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.AGGREGATION_FUNCTIONS_NAME, "", queryissue.AGGREGATE_FUNCTION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SECURITY_INVOKER_VIEWS_NAME, "", queryissue.SECURITY_INVOKER_VIEWS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_TYPE_PREDICATE_NAME, "", queryissue.JSON_TYPE_PREDICATE, schemaAnalysisReport, false, "")) return lo.Filter(unsupportedFeatures, func(f UnsupportedFeature, _ int) bool { return len(f.Objects) > 0 diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index 117e6dbdf..63c89270e 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -51,6 +51,8 @@ const ( AGGREGATE_FUNCTION = "AGGREGATE_FUNCTION" AGGREGATION_FUNCTIONS_NAME = "Aggregate Functions" + JSON_TYPE_PREDICATE = "JSON_TYPE_PREDICATE" + JSON_TYPE_PREDICATE_NAME = "Json Type Predicate" JSON_CONSTRUCTOR_FUNCTION = "JSON_CONSTRUCTOR_FUNCTION" JSON_CONSTRUCTOR_FUNCTION_NAME = "Json Constructor Functions" JSON_QUERY_FUNCTION = "JSON_QUERY_FUNCTION" diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index c834fa6db..d1ec43c5e 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -390,3 +390,35 @@ func (d *JsonQueryFunctionDetector) GetIssues() []QueryIssue { } return issues } + +type JsonPredicateExprDetector struct { + query string + detected bool +} + +func NewJsonPredicateExprDetector(query string) *JsonPredicateExprDetector { + return &JsonPredicateExprDetector{ + query: query, + } +} + +func (j *JsonPredicateExprDetector) Detect(msg protoreflect.Message) error { + if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_JSON_IS_PREDICATE_NODE { + /* + SELECT js IS JSON "json?" FROM (VALUES ('123')) foo(js); + stmts:{stmt:{select_stmt:{target_list:{res_target:{val:{column_ref:{fields:{string:{sval:"js"}} location:337}} location:337}} + target_list:{res_target:{name:"json?" val:{json_is_predicate:{expr:{column_ref:{fields:{string:{sval:"js"}} location:341}} + format:{format_type:JS_FORMAT_DEFAULT encoding:JS_ENC_DEFAULT location:-1} item_type:JS_TYPE_ANY location:341}} location:341}} ... + */ + j.detected = true + } + return nil +} + +func (j *JsonPredicateExprDetector) GetIssues() []QueryIssue { + var issues []QueryIssue + if j.detected { + issues = append(issues, NewJsonPredicateIssue(DML_QUERY_OBJECT_TYPE, "", j.query)) + } + return issues +} diff --git a/yb-voyager/src/query/queryissue/detectors_test.go b/yb-voyager/src/query/queryissue/detectors_test.go index 16c136d3b..e1aa7a32a 100644 --- a/yb-voyager/src/query/queryissue/detectors_test.go +++ b/yb-voyager/src/query/queryissue/detectors_test.go @@ -98,13 +98,6 @@ func TestFuncCallDetector(t *testing.T) { `SELECT pg_advisory_unlock_all();`, } - anyValAggSqls := []string{ - `SELECT - department, - any_value(employee_name) AS any_employee - FROM employees - GROUP BY department;`, - } loFunctionSqls := []string{ `UPDATE documents SET content_oid = lo_import('/path/to/new/file.pdf') @@ -130,13 +123,9 @@ WHERE title = 'Design Document';`, issues := getDetectorIssues(t, NewFuncCallDetector(sql), sql) assert.Equal(t, len(issues), 1) assert.Equal(t, issues[0].Type, LARGE_OBJECT_FUNCTIONS, "Large Objects not detected in SQL: %s", sql) - } - for _, sql := range anyValAggSqls { - issues := getDetectorIssues(t, NewFuncCallDetector(sql), sql) - assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql) - assert.Equal(t, AGGREGATE_FUNCTION, issues[0].Type, "Expected Advisory Locks issue for SQL: %s", sql) } + } func TestColumnRefDetector(t *testing.T) { @@ -710,3 +699,12 @@ WHERE JSON_EXISTS(details, '$.price ? (@ > $price)' PASSING 30 AS price);` assert.Equal(t, JSON_QUERY_FUNCTION, issues[0].Type, "Expected Advisory Locks issue for SQL: %s", sql) } + +func TestIsJsonPredicate(t *testing.T) { + sql := `SELECT js, js IS JSON "json?" FROM (VALUES ('123'), ('"abc"'), ('{"a": "b"}'), ('[1,2]'),('abc')) foo(js);` + + issues := getDetectorIssues(t, NewJsonPredicateExprDetector(sql), sql) + assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql) + assert.Equal(t, JSON_TYPE_PREDICATE, issues[0].Type, "Expected Advisory Locks issue for SQL: %s", sql) + +} diff --git a/yb-voyager/src/query/queryissue/helpers.go b/yb-voyager/src/query/queryissue/helpers.go index d24a14ed5..12f2a2e24 100644 --- a/yb-voyager/src/query/queryissue/helpers.go +++ b/yb-voyager/src/query/queryissue/helpers.go @@ -104,7 +104,7 @@ var UnsupportedIndexDatatypes = []string{ var unsupportedAggFunctions = mapset.NewThreadUnsafeSet([]string{ //agg function added in PG16 - https://www.postgresql.org/docs/16/functions-aggregate.html#id-1.5.8.27.5.2.4.1.1.1.1 - "any_value", + "any_value", "range_agg", "range_intersect_agg", }...) const ( diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index 7e6f8aed8..0752051de 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -74,10 +74,10 @@ func NewRegexFunctionsIssue(objectType string, objectName string, sqlStatement s return newQueryIssue(regexFunctionsIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } -var anyValueAggFunctionIssue = issue.Issue{ +var aggregateFunctionIssue = issue.Issue{ Type: AGGREGATE_FUNCTION, TypeName: AGGREGATION_FUNCTIONS_NAME, - TypeDescription: "Postgresql 17 features not supported yet in YugabyteDB", + TypeDescription: "any_value, range_agg and range_intersect_agg functions not supported yet in YugabyteDB", Suggestion: "", GH: "", DocsLink: "", @@ -88,7 +88,7 @@ func NewAggregationFunctionIssue(objectType string, objectName string, sqlStatem details := map[string]interface{}{ FUNCTION_NAMES: funcNames, //TODO USE it later when we start putting these in reports } - return newQueryIssue(anyValueAggFunctionIssue, objectType, objectName, sqlStatement, details) + return newQueryIssue(aggregateFunctionIssue, objectType, objectName, sqlStatement, details) } var jsonConstructorFunctionsIssue = issue.Issue{ @@ -142,6 +142,19 @@ func NewLOFuntionsIssue(objectType string, objectName string, sqlStatement strin return newQueryIssue(loFunctionsIssue, objectType, objectName, sqlStatement, details) } +var jsonPredicateIssue = issue.Issue{ + Type: JSON_TYPE_PREDICATE, + TypeName: JSON_TYPE_PREDICATE_NAME, + TypeDescription: "IS JSON predicate expressions not supported yet in YugabyteDB", + Suggestion: "", + GH: "", + DocsLink: "", //TODO +} + +func NewJsonPredicateIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(jsonPredicateIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} + var copyFromWhereIssue = issue.Issue{ Type: COPY_FROM_WHERE, TypeName: "COPY FROM ... WHERE", diff --git a/yb-voyager/src/query/queryissue/issues_dml_test.go b/yb-voyager/src/query/queryissue/issues_dml_test.go index cff5910c2..6825eb1af 100644 --- a/yb-voyager/src/query/queryissue/issues_dml_test.go +++ b/yb-voyager/src/query/queryissue/issues_dml_test.go @@ -122,6 +122,17 @@ func testJsonConstructorFunctions(t *testing.T) { } } +func testJsonPredicateIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, `SELECT js, js IS JSON "json?" FROM (VALUES ('123'), ('"abc"'), ('{"a": "b"}'), ('[1,2]'),('abc')) foo(js);`) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, `syntax error at or near "JSON"`, jsonConstructorFunctionsIssue) +} + func testJsonQueryFunctions(t *testing.T) { ctx := context.Background() conn, err := getConn() @@ -159,6 +170,55 @@ WHERE JSON_EXISTS(details, '$.author');`, assertErrorCorrectlyThrownForIssueForYBVersion(t, err, `syntax error at or near "COLUMNS"`, jsonConstructorFunctionsIssue) } +func testAggFunctions(t *testing.T) { + sqls := []string{ + `CREATE TABLE any_value_ex ( + department TEXT, + employee_name TEXT, + salary NUMERIC +); + +INSERT INTO any_value_ex VALUES +('HR', 'Alice', 50000), +('HR', 'Bob', 55000), +('IT', 'Charlie', 60000), +('IT', 'Diana', 62000); + +SELECT + department, + any_value(employee_name) AS any_employee +FROM any_value_ex +GROUP BY department;`, + +`CREATE TABLE events ( + id SERIAL PRIMARY KEY, + event_range daterange +); + +INSERT INTO events (event_range) VALUES + ('[2024-01-01, 2024-01-10]'::daterange), + ('[2024-01-05, 2024-01-15]'::daterange), + ('[2024-01-20, 2024-01-25]'::daterange); + +SELECT range_agg(event_range) AS union_of_ranges +FROM events; + +SELECT range_intersect_agg(event_range) AS intersection_of_ranges +FROM events;`, + } + + for _, sql := range sqls { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, sql) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, `does not exist`, aggregateFunctionIssue) + } +} + func TestDMLIssuesInYBVersion(t *testing.T) { var err error ybVersion := os.Getenv("YB_VERSION") @@ -204,4 +264,10 @@ func TestDMLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "json query functions", ybVersion), testJsonQueryFunctions) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "aggregate functions", ybVersion), testAggFunctions) + assert.True(t, success) + + success = t.Run(fmt.Sprintf("%s-%s", "json type predicate", ybVersion), testJsonPredicateIssue) + assert.True(t, success) + } diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector.go b/yb-voyager/src/query/queryissue/parser_issue_detector.go index c7faf1209..e4db26c0f 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector.go @@ -379,6 +379,7 @@ func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) NewCopyCommandUnsupportedConstructsDetector(query), NewJsonConstructorFuncDetector(query), NewJsonQueryFunctionDetector(query), + NewJsonPredicateExprDetector(query), } processor := func(msg protoreflect.Message) error { diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index c09c4e59d..28c251c78 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -25,9 +25,9 @@ import ( "github.com/google/go-cmp/cmp" "github.com/samber/lo" "github.com/stretchr/testify/assert" - testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) const ( @@ -546,6 +546,19 @@ FROM books;`, `SELECT id, JSON_VALUE(details, '$.title') AS title FROM books WHERE JSON_EXISTS(details, '$.price ? (@ > $price)' PASSING 30 AS price);`, + `SELECT js, js IS JSON "json?", js IS JSON SCALAR "scalar?", js IS JSON OBJECT "object?", js IS JSON ARRAY "array?" +FROM (VALUES ('123'), ('"abc"'), ('{"a": "b"}'), ('[1,2]'),('abc')) foo(js);`, + `SELECT js, + js IS JSON OBJECT "object?", + js IS JSON ARRAY "array?", + js IS JSON ARRAY WITH UNIQUE KEYS "array w. UK?", + js IS JSON ARRAY WITHOUT UNIQUE KEYS "array w/o UK?" +FROM (VALUES ('[{"a":"1"}, + {"b":"2","b":"3"}]')) foo(js);`, + `SELECT js, + js IS JSON OBJECT "object?" + FROM (VALUES ('[{"a":"1"}, + {"b":"2","b":"3"}]')) foo(js); `, `CREATE MATERIALIZED VIEW public.test_jsonb_view AS SELECT id, @@ -560,6 +573,11 @@ JSON_TABLE(data, '$.skills[*]' ) ) AS jt;`, `SELECT JSON_ARRAY($1, 12, TRUE, $2) AS json_array;`, +`CREATE TABLE sales.json_data ( + id int PRIMARY KEY, + array_column TEXT CHECK (array_column IS JSON ARRAY), + unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS) +);`, } sqlsWithExpectedIssues := map[string][]QueryIssue{ sqls[0]: []QueryIssue{ @@ -604,10 +622,22 @@ JSON_TABLE(data, '$.skills[*]' NewJsonQueryFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[12], []string{JSON_VALUE, JSON_EXISTS}), }, sqls[13]: []QueryIssue{ - NewJsonQueryFunctionIssue("MVIEW", "public.test_jsonb_view", sqls[13], []string{JSON_VALUE, JSON_EXISTS, JSON_TABLE}), + NewJsonPredicateIssue(DML_QUERY_OBJECT_TYPE, "", sqls[13]), }, sqls[14]: []QueryIssue{ - NewJsonConstructorFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[14], []string{JSON_ARRAY}), + NewJsonPredicateIssue(DML_QUERY_OBJECT_TYPE, "", sqls[14]), + }, + sqls[15]: []QueryIssue{ + NewJsonPredicateIssue(DML_QUERY_OBJECT_TYPE, "", sqls[15]), + }, + sqls[16]: []QueryIssue{ + NewJsonQueryFunctionIssue("MVIEW", "public.test_jsonb_view", sqls[16], []string{JSON_VALUE, JSON_EXISTS, JSON_TABLE}), + }, + sqls[17]: []QueryIssue{ + NewJsonConstructorFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[17], []string{JSON_ARRAY}), + }, + sqls[18]: []QueryIssue{ + NewJsonPredicateIssue("TABLE", "sales.json_data", sqls[18]), }, } parserIssueDetector := NewParserIssueDetector() @@ -623,6 +653,63 @@ JSON_TABLE(data, '$.skills[*]' } } } + +func TestAggregateFunctions(t *testing.T) { + sqls := []string{ + `SELECT + department, + any_value(employee_name) AS any_employee + FROM employees + GROUP BY department;`, + `SELECT range_intersect_agg(multi_event_range) AS intersection_of_multiranges +FROM multiranges;`, + `SELECT range_agg(multi_event_range) AS union_of_multiranges +FROM multiranges;`, + `CREATE OR REPLACE FUNCTION aggregate_ranges() +RETURNS INT4MULTIRANGE AS $$ +DECLARE + aggregated_range INT4MULTIRANGE; +BEGIN + SELECT range_agg(range_value) INTO aggregated_range FROM ranges; + SELECT + department, + any_value(employee_name) AS any_employee + FROM employees + GROUP BY department; + RETURN aggregated_range; +END; +$$ LANGUAGE plpgsql;`, + } + aggregateSqls := map[string][]QueryIssue{ + sqls[0]: []QueryIssue{ + NewAggregationFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[0], []string{"any_value"}), + }, + sqls[1]: []QueryIssue{ + NewAggregationFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[1], []string{"range_intersect_agg"}), + }, + sqls[2]: []QueryIssue{ + NewAggregationFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[2], []string{"range_agg"}), + }, + sqls[3]: []QueryIssue{ + NewAggregationFunctionIssue(DML_QUERY_OBJECT_TYPE, "", "SELECT range_agg(range_value) FROM ranges;", []string{"range_agg"}), + NewAggregationFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[0], []string{"any_value"}), + }, + } + aggregateSqls[sqls[3]] = modifiedIssuesforPLPGSQL(aggregateSqls[sqls[3]], "FUNCTION", "aggregate_ranges") + + parserIssueDetector := NewParserIssueDetector() + for stmt, expectedIssues := range aggregateSqls { + issues, err := parserIssueDetector.GetAllIssues(stmt, ybversion.LatestStable) + assert.NoError(t, err, "Error detecting issues for statement: %s", stmt) + assert.Equal(t, len(expectedIssues), len(issues), "Mismatch in issue count for statement: %s", stmt) + for _, expectedIssue := range expectedIssues { + found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { + return cmp.Equal(expectedIssue, queryIssue) + }) + assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) + } + } +} func TestRegexFunctionsIssue(t *testing.T) { dmlStmts := []string{ `SELECT regexp_count('This is an example. Another example. Example is a common word.', 'example')`, diff --git a/yb-voyager/src/query/queryparser/traversal_proto.go b/yb-voyager/src/query/queryparser/traversal_proto.go index 9ca3ae3ad..1f18e4565 100644 --- a/yb-voyager/src/query/queryparser/traversal_proto.go +++ b/yb-voyager/src/query/queryparser/traversal_proto.go @@ -49,6 +49,7 @@ const ( PG_QUERY_JSON_FUNC_EXPR_NODE = "pg_query.JsonFuncExpr" PG_QUERY_JSON_OBJECT_CONSTRUCTOR_NODE = "pg_query.JsonObjectConstructor" PG_QUERY_JSON_TABLE_NODE = "pg_query.JsonTable" + PG_QUERY_JSON_IS_PREDICATE_NODE = "pg_query.JsonIsPredicate" PG_QUERY_VIEWSTMT_NODE = "pg_query.ViewStmt" PG_QUERY_COPYSTSMT_NODE = "pg_query.CopyStmt" )