Reporting range aggregate functions and IS JSON predicate clauses

migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql

@@ -386,6 +386,15 @@ CREATE TABLE public.locations (
 
 CREATE TABLE image (title text, raster lo);
 
+-- IS JSON Predicate
+CREATE TABLE public.json_data (
+    id SERIAL PRIMARY KEY,
+    data_column TEXT NOT NULL CHECK (data_column IS JSON),
+    object_column TEXT CHECK (object_column IS JSON OBJECT),
+    array_column TEXT CHECK (array_column IS JSON ARRAY),
+    scalar_column TEXT CHECK (scalar_column IS JSON SCALAR),
+    unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS)
+);
 CREATE TABLE employees (id INT PRIMARY KEY, salary INT);
 -- create table with multirange data types
 

migtests/tests/analyze-schema/expected_issues.json

@@ -30,6 +30,16 @@
             "GH": "",
             "MinimumVersionsFixedIn": null
         },
+        {
+            "IssueType": "unsupported_features",
+            "ObjectType": "TABLE",
+            "ObjectName": "public.json_data",
+            "Reason": "Json Type Predicate",
+            "SqlStatement": "CREATE TABLE public.json_data (\n    id SERIAL PRIMARY KEY,\n    data_column TEXT NOT NULL CHECK (data_column IS JSON),\n    object_column TEXT CHECK (object_column IS JSON OBJECT),\n    array_column TEXT CHECK (array_column IS JSON ARRAY),\n    scalar_column TEXT CHECK (scalar_column IS JSON SCALAR),\n    unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS)\n);",
+            "Suggestion": "",
+            "GH": "",
+            "MinimumVersionsFixedIn": null
+        },
         {
             "IssueType": "unsupported_features",
             "ObjectType": "VIEW",

migtests/tests/analyze-schema/summary.json

@@ -26,9 +26,9 @@
             },
             {
                 "ObjectType": "TABLE",
-                "TotalCount": 58,
-                "InvalidCount": 49,
-                "ObjectNames": "employees, image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table"            },
+                "TotalCount": 59,
+                "InvalidCount": 50,
+                "ObjectNames": "public.json_data, employees, image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table"            },
 
             {
                 "ObjectType": "INDEX",

migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json

@@ -25,25 +25,27 @@
 			},
 			{
 				"ObjectType": "TABLE",
-				"TotalCount": 2,
-				"InvalidCount": 0,
-				"ObjectNames": "analytics.metrics, sales.orders"
+				"TotalCount": 4,
+				"InvalidCount": 1,
+				"ObjectNames": "sales.json_data, analytics.metrics, sales.events, sales.orders"
 			},
 			{
 				"ObjectType": "VIEW",
-				"TotalCount": 1,
-				"InvalidCount": 1,
-				"ObjectNames": "sales.employ_depart_view"
+				"TotalCount": 2,
+				"InvalidCount": 2,
+				"ObjectNames": "sales.event_analysis_view, sales.employ_depart_view"
 			}
 		]
 	},
 	"Sizing": {
 		"SizingRecommendation": {
 			"ColocatedTables": [
 				"sales.orders",
-				"analytics.metrics"
+				"analytics.metrics",
+				"sales.events",
+				"sales.json_data"
 			],
-			"ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 2 objects (2 tables/materialized views and 0 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.",
+			"ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 4 objects (4 tables/materialized views and 0 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.",
 			"ShardedTables": null,
 			"NumNodes": 3,
 			"VCPUsPerInstance": 4,
@@ -55,18 +57,39 @@
 		},
 		"FailureReasoning": ""
 	},
-	"UnsupportedDataTypes": null,
+	"UnsupportedDataTypes": [
+		{
+			"SchemaName": "sales",
+			"TableName": "event_analysis_view",
+			"ColumnName": "all_event_ranges",
+			"DataType": "datemultirange"
+		}
+	],
 	"UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.",
 	"UnsupportedFeatures": [
 		{
 			"FeatureName": "Aggregate Functions",
 			"Objects": [
+				{
+					"ObjectName": "sales.event_analysis_view",
+					"SqlStatement": "CREATE VIEW sales.event_analysis_view AS\n SELECT range_agg(event_range) AS all_event_ranges,\n    range_intersect_agg(event_range) AS overlapping_range,\n    lower(range_agg(event_range)) AS earliest_start,\n    upper(range_agg(event_range)) AS latest_end\n   FROM sales.events;"
+				},
 				{
 					"ObjectName": "sales.employ_depart_view",
 					"SqlStatement": "CREATE VIEW sales.employ_depart_view AS\n SELECT any_value(name) AS any_employee\n   FROM public.employees;"
 				}
 			],
 			"MinimumVersionsFixedIn": null
+		},
+		{
+			"FeatureName": "Json Type Predicate",
+			"Objects": [
+				{
+					"ObjectName": "sales.json_data",
+					"SqlStatement": "CREATE TABLE sales.json_data (\n    id integer NOT NULL,\n    data_column text NOT NULL,\n    object_column text,\n    array_column text,\n    scalar_column text,\n    unique_keys_column text,\n    CONSTRAINT json_data_array_column_check CHECK ((array_column IS JSON ARRAY)),\n    CONSTRAINT json_data_data_column_check CHECK ((data_column IS JSON)),\n    CONSTRAINT json_data_object_column_check CHECK ((object_column IS JSON OBJECT)),\n    CONSTRAINT json_data_scalar_column_check CHECK ((scalar_column IS JSON SCALAR)),\n    CONSTRAINT json_data_unique_keys_column_check CHECK ((unique_keys_column IS JSON WITH UNIQUE KEYS))\n);"
+				}
+			],
+			"MinimumVersionsFixedIn": null
 		}
 	],
 	"UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.",
@@ -85,6 +108,34 @@
 			"ParentTableName": null,
 			"SizeInBytes": 8192
 		},
+		{
+			"SchemaName": "sales",
+			"ObjectName": "events",
+			"RowCount": 3,
+			"ColumnCount": 2,
+			"Reads": 6,
+			"Writes": 3,
+			"ReadsPerSecond": 0,
+			"WritesPerSecond": 0,
+			"IsIndex": false,
+			"ObjectType": "",
+			"ParentTableName": null,
+			"SizeInBytes": 8192
+		},
+		{
+			"SchemaName": "sales",
+			"ObjectName": "json_data",
+			"RowCount": 0,
+			"ColumnCount": 6,
+			"Reads": 0,
+			"Writes": 0,
+			"ReadsPerSecond": 0,
+			"WritesPerSecond": 0,
+			"IsIndex": false,
+			"ObjectType": "",
+			"ParentTableName": null,
+			"SizeInBytes": 0
+		},
 		{
 			"SchemaName": "analytics",
 			"ObjectName": "metrics",
@@ -109,11 +160,29 @@
 			"DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented",
 			"MinimumVersionsFixedIn": null
 		},
+		{
+			"ConstructTypeName": "Aggregate Functions",
+			"Query": "SELECT range_intersect_agg(event_range) AS intersection_of_ranges\nFROM sales.events",
+			"DocsLink": "",
+			"MinimumVersionsFixedIn": null
+		},
+		{
+			"ConstructTypeName": "Aggregate Functions",
+			"Query": "SELECT range_agg(event_range) AS union_of_ranges\nFROM sales.events",
+			"DocsLink": "",
+			"MinimumVersionsFixedIn": null
+		},
 		{
 			"ConstructTypeName": "Aggregate Functions",
 			"Query": "SELECT\n        any_value(name) AS any_employee\n    FROM employees",
 			"DocsLink": "",
 			"MinimumVersionsFixedIn": null
+		},
+		{
+			"ConstructTypeName": "Json Type Predicate",
+			"Query": "SELECT * \nFROM sales.json_data\nWHERE array_column IS JSON ARRAY",
+			"DocsLink": "",
+			"MinimumVersionsFixedIn": null
 		}
 	],
 	"UnsupportedPlPgSqlObjects": null

migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql

@@ -45,4 +45,44 @@ INSERT INTO analytics.metrics (metric_name, metric_value) VALUES ('ConversionRat
 
 create view sales.employ_depart_view AS  SELECT
         any_value(name) AS any_employee
-    FROM employees;
+    FROM employees;
+
+CREATE TABLE sales.events (
+    id int PRIMARY KEY,
+    event_range daterange
+);
+
+-- Insert some ranges
+INSERT INTO sales.events (id, event_range) VALUES
+    (1,'[2024-01-01, 2024-01-10]'::daterange),
+    (2,'[2024-01-05, 2024-01-15]'::daterange),
+    (3,'[2024-01-20, 2024-01-25]'::daterange);
+
+CREATE VIEW sales.event_analysis_view AS
+SELECT
+    range_agg(event_range) AS all_event_ranges, 
+    range_intersect_agg(event_range) AS overlapping_range, 
+    lower(range_agg(event_range)) AS earliest_start, 
+    upper(range_agg(event_range)) AS latest_end 
+FROM
+    sales.events;
+
+-- PG 16 and above feature 
+CREATE TABLE sales.json_data (
+    id int PRIMARY KEY,
+    data_column TEXT NOT NULL CHECK (data_column IS JSON),
+    object_column TEXT CHECK (object_column IS JSON OBJECT),
+    array_column TEXT CHECK (array_column IS JSON ARRAY),
+    scalar_column TEXT CHECK (scalar_column IS JSON SCALAR),
+    unique_keys_column TEXT CHECK (unique_keys_column IS JSON WITH UNIQUE KEYS)
+);
+
+INSERT INTO public.json_data (
+    id, data_column, object_column, array_column, scalar_column, unique_keys_column
+) VALUES (
+    1, '{"key": "value"}',
+    2, '{"name": "John", "age": 30}',
+    3, '[1, 2, 3, 4]',
+    4, '"hello"',
+    5, '{"uniqueKey1": "value1", "uniqueKey2": "value2"}'
+);

migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql

@@ -25,4 +25,16 @@ WHERE metric_value > 0.02;
 -- Aggregate functions UQC NOT REPORTING as it need PG16 upgarde in pipeline from PG15
 SELECT
         any_value(name) AS any_employee
-    FROM employees;
+    FROM employees;
+
+--PG15
+SELECT range_agg(event_range) AS union_of_ranges
+FROM sales.events;
+
+SELECT range_intersect_agg(event_range) AS intersection_of_ranges
+FROM sales.events;
+
+-- -- PG 16 and above feature
+SELECT * 
+FROM sales.json_data
+WHERE array_column IS JSON ARRAY;

yb-voyager/cmd/assessMigrationCommand.go

@@ -1028,6 +1028,7 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem
 	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_CONSTRUCTOR_FUNCTION_NAME, "", queryissue.JSON_CONSTRUCTOR_FUNCTION, schemaAnalysisReport, false, ""))
 	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.AGGREGATION_FUNCTIONS_NAME, "", queryissue.AGGREGATE_FUNCTION, schemaAnalysisReport, false, ""))
 	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SECURITY_INVOKER_VIEWS_NAME, "", queryissue.SECURITY_INVOKER_VIEWS, schemaAnalysisReport, false, ""))
+	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_TYPE_PREDICATE_NAME, "", queryissue.JSON_TYPE_PREDICATE, schemaAnalysisReport, false, ""))
 
 	return lo.Filter(unsupportedFeatures, func(f UnsupportedFeature, _ int) bool {
 		return len(f.Objects) > 0

yb-voyager/src/query/queryissue/constants.go

@@ -51,6 +51,8 @@ const (
 
 	AGGREGATE_FUNCTION             = "AGGREGATE_FUNCTION"
 	AGGREGATION_FUNCTIONS_NAME     = "Aggregate Functions"
+	JSON_TYPE_PREDICATE            = "JSON_TYPE_PREDICATE"
+	JSON_TYPE_PREDICATE_NAME       = "Json Type Predicate"
 	JSON_CONSTRUCTOR_FUNCTION      = "JSON_CONSTRUCTOR_FUNCTION"
 	JSON_CONSTRUCTOR_FUNCTION_NAME = "Json Constructor Functions"
 	JSON_QUERY_FUNCTION            = "JSON_QUERY_FUNCTION"

yb-voyager/src/query/queryissue/detectors.go

@@ -390,3 +390,35 @@ func (d *JsonQueryFunctionDetector) GetIssues() []QueryIssue {
 	}
 	return issues
 }
+
+type JsonPredicateExprDetector struct {
+	query    string
+	detected bool
+}
+
+func NewJsonPredicateExprDetector(query string) *JsonPredicateExprDetector {
+	return &JsonPredicateExprDetector{
+		query: query,
+	}
+}
+
+func (j *JsonPredicateExprDetector) Detect(msg protoreflect.Message) error {
+	if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_JSON_PREDICATE_NODE {
+		/*
+		SELECT  js IS JSON "json?" FROM (VALUES ('123')) foo(js);
+		stmts:{stmt:{select_stmt:{target_list:{res_target:{val:{column_ref:{fields:{string:{sval:"js"}}  location:337}}  location:337}} 
+		target_list:{res_target:{name:"json?"  val:{json_is_predicate:{expr:{column_ref:{fields:{string:{sval:"js"}}  location:341}}  
+		format:{format_type:JS_FORMAT_DEFAULT  encoding:JS_ENC_DEFAULT  location:-1}  item_type:JS_TYPE_ANY  location:341}}  location:341}} ...
+		*/
+		j.detected = true
+	}
+	return nil
+}
+
+func (d *JsonPredicateExprDetector) GetIssues() []QueryIssue {
+	var issues []QueryIssue
+	if d.detected {
+		issues = append(issues, NewJsonPredicateIssue(DML_QUERY_OBJECT_TYPE, "", d.query))
+	}
+	return issues
+}

yb-voyager/src/query/queryissue/detectors_test.go

@@ -98,13 +98,6 @@ func TestFuncCallDetector(t *testing.T) {
 		`SELECT pg_advisory_unlock_all();`,
 	}
 
-	anyValAggSqls := []string{
-		`SELECT
-		department,
-		any_value(employee_name) AS any_employee
-	FROM employees
-	GROUP BY department;`,
-	}
 	loFunctionSqls := []string{
 		`UPDATE documents
 SET content_oid = lo_import('/path/to/new/file.pdf')
@@ -130,13 +123,9 @@ WHERE title = 'Design Document';`,
 		issues := getDetectorIssues(t, NewFuncCallDetector(sql), sql)
 		assert.Equal(t, len(issues), 1)
 		assert.Equal(t, issues[0].Type, LARGE_OBJECT_FUNCTIONS, "Large Objects not detected in SQL: %s", sql)
-	}
 
-	for _, sql := range anyValAggSqls {
-		issues := getDetectorIssues(t, NewFuncCallDetector(sql), sql)
-		assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql)
-		assert.Equal(t, AGGREGATE_FUNCTION, issues[0].Type, "Expected Advisory Locks issue for SQL: %s", sql)
 	}
+
 }
 
 func TestColumnRefDetector(t *testing.T) {
@@ -710,3 +699,12 @@ WHERE JSON_EXISTS(details, '$.price ? (@ > $price)' PASSING 30 AS price);`
 	assert.Equal(t, JSON_QUERY_FUNCTION, issues[0].Type, "Expected Advisory Locks issue for SQL: %s", sql)
 
 }
+
+func TestIsJsonPredicate(t *testing.T) {
+	sql := `SELECT js, js IS JSON "json?" FROM (VALUES ('123'), ('"abc"'), ('{"a": "b"}'), ('[1,2]'),('abc')) foo(js);`
+
+	issues := getDetectorIssues(t, NewJsonPredicateExprDetector(sql), sql)
+	assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql)
+	assert.Equal(t, JSON_TYPE_PREDICATE, issues[0].Type, "Expected Advisory Locks issue for SQL: %s", sql)
+
+}

yb-voyager/src/query/queryissue/helpers.go

@@ -104,7 +104,7 @@ var UnsupportedIndexDatatypes = []string{
 
 var unsupportedAggFunctions = mapset.NewThreadUnsafeSet([]string{
 	//agg function added in PG16 - https://www.postgresql.org/docs/16/functions-aggregate.html#id-1.5.8.27.5.2.4.1.1.1.1
-	"any_value",
+	"any_value", "range_agg", "range_intersect_agg",
 }...)
 
 const (