Reporting range aggregate functions and IS JSON predicate clauses (#2113

) Reporting the IS JSON predicate clauses - https://yugabyte.atlassian.net/browse/DB-14543 Reporting the range_agg, range_intersect_agg - range type functions - https://yugabyte.atlassian.net/browse/DB-14224
yugabyte · Jan 6, 2025 · 16b08fe · 16b08fe
1 parent 953591c
commit 16b08fe
Show file tree

Hide file tree

Showing 16 changed files with 428 additions and 43 deletions.
diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql
@@ -362,30 +362,39 @@ CREATE TABLE test_udt (
 	employee_name VARCHAR(100),
 	home_address address_type,
 	some_field enum_test,
-	home_address1 non_public.address_type1
+	home_address1 non_public.address_type1,
+    scalar_column TEXT CHECK (scalar_column IS JSON SCALAR)
 );
 
 CREATE TABLE test_arr_enum (
 	id int,
 	arr text[],
-	arr_enum enum_test[]
+	arr_enum enum_test[],
+    object_column TEXT CHECK (object_column IS JSON OBJECT)
 );
 
 CREATE TABLE public.locations (
     id integer NOT NULL,
     name character varying(100),
-    geom geometry(Point,4326)
+    geom geometry(Point,4326),
+    array_column TEXT CHECK (array_column IS JSON ARRAY)
  );
 
  CREATE TABLE public.xml_data_example (
     id SERIAL PRIMARY KEY,
     name VARCHAR(255),
     description XML DEFAULT xmlparse(document '<product><name>Default Product</name><price>100.00</price><category>Electronics</category></product>'),
-    created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
+    created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
+    unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS)
 );
 
 CREATE TABLE image (title text, raster lo);
 
+-- IS JSON Predicate
+CREATE TABLE public.json_data (
+    id SERIAL PRIMARY KEY,
+    data_column TEXT NOT NULL CHECK (data_column IS JSON)
+);
 CREATE TABLE employees (id INT PRIMARY KEY, salary INT);
 -- create table with multirange data types
 

diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json
@@ -30,6 +30,56 @@
             "GH": "",
             "MinimumVersionsFixedIn": null
         },
+        {
+            "IssueType": "unsupported_features",
+            "ObjectType": "TABLE",
+            "ObjectName": "public.json_data",
+            "Reason": "Json Type Predicate",
+            "SqlStatement": "CREATE TABLE public.json_data (\n    id SERIAL PRIMARY KEY,\n    data_column TEXT NOT NULL CHECK (data_column IS JSON)\n);",
+            "Suggestion": "",
+            "GH": "",
+            "MinimumVersionsFixedIn": null
+        },
+        {
+            "IssueType": "unsupported_features",
+            "ObjectType": "TABLE",
+            "ObjectName": "test_arr_enum",
+            "Reason": "Json Type Predicate",
+            "SqlStatement": "CREATE TABLE test_arr_enum (\n\tid int,\n\tarr text[],\n\tarr_enum enum_test[],\n    object_column TEXT CHECK (object_column IS JSON OBJECT)\n);",
+            "Suggestion": "",
+            "GH": "",
+            "MinimumVersionsFixedIn": null
+        },
+        {
+            "IssueType": "unsupported_features",
+            "ObjectType": "TABLE",
+            "ObjectName": "test_udt",
+            "Reason": "Json Type Predicate",
+            "SqlStatement": "CREATE TABLE test_udt (\n\temployee_id SERIAL PRIMARY KEY,\n\temployee_name VARCHAR(100),\n\thome_address address_type,\n\tsome_field enum_test,\n\thome_address1 non_public.address_type1,\n    scalar_column TEXT CHECK (scalar_column IS JSON SCALAR)\n);",
+            "Suggestion": "",
+            "GH": "",
+            "MinimumVersionsFixedIn": null
+        },
+        {
+            "IssueType": "unsupported_features",
+            "ObjectType": "TABLE",
+            "ObjectName": "public.xml_data_example",
+            "Reason": "Json Type Predicate",
+            "SqlStatement": " CREATE TABLE public.xml_data_example (\n    id SERIAL PRIMARY KEY,\n    name VARCHAR(255),\n    description XML DEFAULT xmlparse(document '\u003cproduct\u003e\u003cname\u003eDefault Product\u003c/name\u003e\u003cprice\u003e100.00\u003c/price\u003e\u003ccategory\u003eElectronics\u003c/category\u003e\u003c/product\u003e'),\n    created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,\n    unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS)\n);",
+            "Suggestion": "",
+            "GH": "",
+            "MinimumVersionsFixedIn": null
+        },
+        {
+            "IssueType": "unsupported_features",
+            "ObjectType": "TABLE",
+            "ObjectName": "public.locations",
+            "Reason": "Json Type Predicate",
+            "SqlStatement": "CREATE TABLE public.locations (\n    id integer NOT NULL,\n    name character varying(100),\n    geom geometry(Point,4326),\n    array_column TEXT CHECK (array_column IS JSON ARRAY)\n );",
+            "Suggestion": "",
+            "GH": "",
+            "MinimumVersionsFixedIn": null
+        },
         {
             "IssueType": "unsupported_features",
             "ObjectType": "VIEW",
@@ -55,7 +105,7 @@
             "ObjectType": "TABLE",
             "ObjectName": "public.xml_data_example",
             "Reason": "XML Functions",
-            "SqlStatement": " CREATE TABLE public.xml_data_example (\n    id SERIAL PRIMARY KEY,\n    name VARCHAR(255),\n    description XML DEFAULT xmlparse(document '\u003cproduct\u003e\u003cname\u003eDefault Product\u003c/name\u003e\u003cprice\u003e100.00\u003c/price\u003e\u003ccategory\u003eElectronics\u003c/category\u003e\u003c/product\u003e'),\n    created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP\n);",
+            "SqlStatement": " CREATE TABLE public.xml_data_example (\n    id SERIAL PRIMARY KEY,\n    name VARCHAR(255),\n    description XML DEFAULT xmlparse(document '\u003cproduct\u003e\u003cname\u003eDefault Product\u003c/name\u003e\u003cprice\u003e100.00\u003c/price\u003e\u003ccategory\u003eElectronics\u003c/category\u003e\u003c/product\u003e'),\n    created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,\n    unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS)\n);",
             "Suggestion": "",
             "GH": "",
             "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported",
@@ -66,7 +116,7 @@
             "ObjectType": "TABLE",
             "ObjectName": "public.xml_data_example",
             "Reason": "Unsupported datatype - xml on column - description",
-            "SqlStatement": " CREATE TABLE public.xml_data_example (\n    id SERIAL PRIMARY KEY,\n    name VARCHAR(255),\n    description XML DEFAULT xmlparse(document '\u003cproduct\u003e\u003cname\u003eDefault Product\u003c/name\u003e\u003cprice\u003e100.00\u003c/price\u003e\u003ccategory\u003eElectronics\u003c/category\u003e\u003c/product\u003e'),\n    created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP\n);",
+            "SqlStatement": " CREATE TABLE public.xml_data_example (\n    id SERIAL PRIMARY KEY,\n    name VARCHAR(255),\n    description XML DEFAULT xmlparse(document '\u003cproduct\u003e\u003cname\u003eDefault Product\u003c/name\u003e\u003cprice\u003e100.00\u003c/price\u003e\u003ccategory\u003eElectronics\u003c/category\u003e\u003c/product\u003e'),\n    created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,\n    unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS)\n);",
             "Suggestion": "Data ingestion is not supported for this type in YugabyteDB so handle this type in different way. Refer link for more details.",
             "GH": "https://github.com/yugabyte/yugabyte-db/issues/1043",
             "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported",
@@ -275,7 +325,7 @@
             "ObjectType": "TABLE",
             "ObjectName": "test_udt",
             "Reason": "Unsupported datatype for Live migration with fall-forward/fallback - address_type on column - home_address",
-            "SqlStatement": "CREATE TABLE test_udt (\n\temployee_id SERIAL PRIMARY KEY,\n\temployee_name VARCHAR(100),\n\thome_address address_type,\n\tsome_field enum_test,\n\thome_address1 non_public.address_type1\n);",
+            "SqlStatement": "CREATE TABLE test_udt (\n\temployee_id SERIAL PRIMARY KEY,\n\temployee_name VARCHAR(100),\n\thome_address address_type,\n\tsome_field enum_test,\n\thome_address1 non_public.address_type1,\n    scalar_column TEXT CHECK (scalar_column IS JSON SCALAR)\n);",
             "Suggestion": "",
             "GH": "https://github.com/yugabyte/yb-voyager/issues/1731",
             "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration",
@@ -286,7 +336,7 @@
             "ObjectType": "TABLE",
             "ObjectName": "test_arr_enum",
             "Reason": "Unsupported datatype for Live migration with fall-forward/fallback - enum_test[] on column - arr_enum",
-            "SqlStatement": "CREATE TABLE test_arr_enum (\n\tid int,\n\tarr text[],\n\tarr_enum enum_test[]\n);",
+            "SqlStatement": "CREATE TABLE test_arr_enum (\n\tid int,\n\tarr text[],\n\tarr_enum enum_test[],\n    object_column TEXT CHECK (object_column IS JSON OBJECT)\n);",
             "Suggestion": "",
             "GH": "https://github.com/yugabyte/yb-voyager/issues/1731",
             "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration",
@@ -297,7 +347,7 @@
             "ObjectType": "TABLE",
             "ObjectName": "test_udt",
             "Reason": "Unsupported datatype for Live migration with fall-forward/fallback - non_public.address_type1 on column - home_address1",
-            "SqlStatement": "CREATE TABLE test_udt (\n\temployee_id SERIAL PRIMARY KEY,\n\temployee_name VARCHAR(100),\n\thome_address address_type,\n\tsome_field enum_test,\n\thome_address1 non_public.address_type1\n);",
+            "SqlStatement": "CREATE TABLE test_udt (\n\temployee_id SERIAL PRIMARY KEY,\n\temployee_name VARCHAR(100),\n\thome_address address_type,\n\tsome_field enum_test,\n\thome_address1 non_public.address_type1,\n    scalar_column TEXT CHECK (scalar_column IS JSON SCALAR)\n);",
             "Suggestion": "",
             "GH": "https://github.com/yugabyte/yb-voyager/issues/1731",
             "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration",
@@ -1212,7 +1262,7 @@
             "ObjectType": "TABLE",
             "ObjectName": "public.locations",
             "Reason": "Unsupported datatype - geometry on column - geom",
-            "SqlStatement": "CREATE TABLE public.locations (\n    id integer NOT NULL,\n    name character varying(100),\n    geom geometry(Point,4326)\n );",
+            "SqlStatement": "CREATE TABLE public.locations (\n    id integer NOT NULL,\n    name character varying(100),\n    geom geometry(Point,4326),\n    array_column TEXT CHECK (array_column IS JSON ARRAY)\n );",
             "Suggestion": "",
             "GH": "https://github.com/yugabyte/yugabyte-db/issues/11323",
             "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb",

diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json
@@ -26,9 +26,9 @@
             },
             {
                 "ObjectType": "TABLE",
-                "TotalCount": 58,
-                "InvalidCount": 49,
-                "ObjectNames": "employees, image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table"            },
+                "TotalCount": 59,
+                "InvalidCount": 50,
+                "ObjectNames": "public.json_data, employees, image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table"            },
 
             {
                 "ObjectType": "INDEX",

diff --git a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json
@@ -25,25 +25,27 @@
 			},
 			{
 				"ObjectType": "TABLE",
-				"TotalCount": 2,
-				"InvalidCount": 0,
-				"ObjectNames": "analytics.metrics, sales.orders"
+				"TotalCount": 4,
+				"InvalidCount": 1,
+				"ObjectNames": "sales.json_data, analytics.metrics, sales.events, sales.orders"
 			},
 			{
 				"ObjectType": "VIEW",
-				"TotalCount": 1,
-				"InvalidCount": 1,
-				"ObjectNames": "sales.employ_depart_view"
+				"TotalCount": 3,
+				"InvalidCount": 3,
+				"ObjectNames": "sales.event_analysis_view, sales.event_analysis_view2, sales.employ_depart_view"
 			}
 		]
 	},
 	"Sizing": {
 		"SizingRecommendation": {
 			"ColocatedTables": [
 				"sales.orders",
-				"analytics.metrics"
+				"analytics.metrics",
+				"sales.events",
+				"sales.json_data"
 			],
-			"ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 2 objects (2 tables/materialized views and 0 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.",
+			"ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 4 objects (4 tables/materialized views and 0 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.",
 			"ShardedTables": null,
 			"NumNodes": 3,
 			"VCPUsPerInstance": 4,
@@ -55,18 +57,43 @@
 		},
 		"FailureReasoning": ""
 	},
-	"UnsupportedDataTypes": null,
+	"UnsupportedDataTypes": [
+		{
+			"SchemaName": "sales",
+			"TableName": "event_analysis_view",
+			"ColumnName": "all_event_ranges",
+			"DataType": "datemultirange"
+		}
+	],
 	"UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.",
 	"UnsupportedFeatures": [
 		{
 			"FeatureName": "Aggregate Functions",
 			"Objects": [
+				{
+					"ObjectName": "sales.event_analysis_view",
+					"SqlStatement": "CREATE VIEW sales.event_analysis_view AS\n SELECT range_agg(event_range) AS all_event_ranges\n   FROM sales.events;"
+				},
+				{
+					"ObjectName": "sales.event_analysis_view2",
+					"SqlStatement": "CREATE VIEW sales.event_analysis_view2 AS\n SELECT range_intersect_agg(event_range) AS overlapping_range\n   FROM sales.events;"
+				},
 				{
 					"ObjectName": "sales.employ_depart_view",
 					"SqlStatement": "CREATE VIEW sales.employ_depart_view AS\n SELECT any_value(name) AS any_employee\n   FROM public.employees;"
 				}
 			],
 			"MinimumVersionsFixedIn": null
+		},
+		{
+			"FeatureName": "Json Type Predicate",
+			"Objects": [
+				{
+					"ObjectName": "sales.json_data",
+					"SqlStatement": "CREATE TABLE sales.json_data (\n    id integer NOT NULL,\n    array_column text,\n    unique_keys_column text,\n    CONSTRAINT json_data_array_column_check CHECK ((array_column IS JSON ARRAY)),\n    CONSTRAINT json_data_unique_keys_column_check CHECK ((unique_keys_column IS JSON WITH UNIQUE KEYS))\n);"
+				}
+			],
+			"MinimumVersionsFixedIn": null
 		}
 	],
 	"UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.",
@@ -85,6 +112,34 @@
 			"ParentTableName": null,
 			"SizeInBytes": 8192
 		},
+		{
+			"SchemaName": "sales",
+			"ObjectName": "events",
+			"RowCount": 3,
+			"ColumnCount": 2,
+			"Reads": 6,
+			"Writes": 3,
+			"ReadsPerSecond": 0,
+			"WritesPerSecond": 0,
+			"IsIndex": false,
+			"ObjectType": "",
+			"ParentTableName": null,
+			"SizeInBytes": 8192
+		},
+		{
+			"SchemaName": "sales",
+			"ObjectName": "json_data",
+			"RowCount": 0,
+			"ColumnCount": 3,
+			"Reads": 0,
+			"Writes": 0,
+			"ReadsPerSecond": 0,
+			"WritesPerSecond": 0,
+			"IsIndex": false,
+			"ObjectType": "",
+			"ParentTableName": null,
+			"SizeInBytes": 0
+		},
 		{
 			"SchemaName": "analytics",
 			"ObjectName": "metrics",
@@ -109,11 +164,29 @@
 			"DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented",
 			"MinimumVersionsFixedIn": null
 		},
+		{
+			"ConstructTypeName": "Aggregate Functions",
+			"Query": "SELECT range_intersect_agg(event_range) AS intersection_of_ranges\nFROM sales.events",
+			"DocsLink": "",
+			"MinimumVersionsFixedIn": null
+		},
+		{
+			"ConstructTypeName": "Aggregate Functions",
+			"Query": "SELECT range_agg(event_range) AS union_of_ranges\nFROM sales.events",
+			"DocsLink": "",
+			"MinimumVersionsFixedIn": null
+		},
 		{
 			"ConstructTypeName": "Aggregate Functions",
 			"Query": "SELECT\n        any_value(name) AS any_employee\n    FROM employees",
 			"DocsLink": "",
 			"MinimumVersionsFixedIn": null
+		},
+		{
+			"ConstructTypeName": "Json Type Predicate",
+			"Query": "SELECT * \nFROM sales.json_data\nWHERE array_column IS JSON ARRAY",
+			"DocsLink": "",
+			"MinimumVersionsFixedIn": null
 		}
 	],
 	"UnsupportedPlPgSqlObjects": null

diff --git a/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql b/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql
@@ -45,4 +45,44 @@ INSERT INTO analytics.metrics (metric_name, metric_value) VALUES ('ConversionRat
 
 create view sales.employ_depart_view AS  SELECT
         any_value(name) AS any_employee
-    FROM employees;
+    FROM employees;
+
+CREATE TABLE sales.events (
+    id int PRIMARY KEY,
+    event_range daterange
+);
+
+-- Insert some ranges
+INSERT INTO sales.events (id, event_range) VALUES
+    (1,'[2024-01-01, 2024-01-10]'::daterange),
+    (2,'[2024-01-05, 2024-01-15]'::daterange),
+    (3,'[2024-01-20, 2024-01-25]'::daterange);
+
+CREATE VIEW sales.event_analysis_view AS
+SELECT
+    range_agg(event_range) AS all_event_ranges
+FROM
+    sales.events;
+
+CREATE VIEW sales.event_analysis_view2 AS
+SELECT
+    range_intersect_agg(event_range) AS overlapping_range
+FROM
+    sales.events;
+
+-- PG 16 and above feature 
+CREATE TABLE sales.json_data (
+    id int PRIMARY KEY,
+    array_column TEXT CHECK (array_column IS JSON ARRAY),
+    unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS)
+);
+
+INSERT INTO public.json_data (
+    id, data_column, object_column, array_column, scalar_column, unique_keys_column
+) VALUES (
+    1, '{"key": "value"}',
+    2, '{"name": "John", "age": 30}',
+    3, '[1, 2, 3, 4]',
+    4, '"hello"',
+    5, '{"uniqueKey1": "value1", "uniqueKey2": "value2"}'
+);
diff --git a/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql b/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql
@@ -25,4 +25,16 @@ WHERE metric_value > 0.02;
 -- Aggregate functions UQC NOT REPORTING as it need PG16 upgarde in pipeline from PG15
 SELECT
         any_value(name) AS any_employee
-    FROM employees;
+    FROM employees;
+
+--PG15
+SELECT range_agg(event_range) AS union_of_ranges
+FROM sales.events;
+
+SELECT range_intersect_agg(event_range) AS intersection_of_ranges
+FROM sales.events;
+
+-- -- PG 16 and above feature
+SELECT * 
+FROM sales.json_data
+WHERE array_column IS JSON ARRAY;