Reporting the Advisory locks, XML Functions and System columns on the…

… DDLs in the unsupported features in assess/analyze (#2061) - Added three more features to `Unsupported features` in the reports Advisory locks, XML functions and System columns in the DDL. - Moved the MVIEW/VIEW issues from Unsupported PL/pgSQL objects to Unsupported Features. - Minor format change of constraint reporting for PK/Unique on complex datatypes issue - Added unit tests for `GetDDLIssues()` in the `parser_issue_detector_test.go` Fixes #2025
yugabyte · Dec 13, 2024 · 13a678b · 13a678b
1 parent 5fec48b
commit 13a678b
Show file tree

Hide file tree

Showing 20 changed files with 523 additions and 262 deletions.
diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql
@@ -375,4 +375,11 @@ CREATE TABLE public.locations (
     id integer NOT NULL,
     name character varying(100),
     geom geometry(Point,4326)
- );
+ );
+
+ CREATE TABLE public.xml_data_example (
+    id SERIAL PRIMARY KEY,
+    name VARCHAR(255),
+    description XML DEFAULT xmlparse(document '<product><name>Default Product</name><price>100.00</price><category>Electronics</category></product>'),
+    created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
+);
diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json
diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json
@@ -24,9 +24,9 @@
             },
             {
                 "ObjectType": "TABLE",
-                "TotalCount": 49,
+                "TotalCount": 50,
                 "InvalidCount": 33,
-                "ObjectNames": "combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr"            },
+                "ObjectNames": "public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr"            },
             {
                 "ObjectType": "INDEX",
                 "TotalCount": 43,

diff --git a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json
diff --git a/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json
diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json
@@ -522,15 +522,15 @@
 			"FeatureName": "Primary / Unique key constraints on complex datatypes",
 			"Objects": [
 				{
-					"ObjectName": "public.combined_tbl, constraint: combined_tbl_bittv_key",
+					"ObjectName": "public.combined_tbl, constraint: (combined_tbl_bittv_key)",
 					"SqlStatement": "ALTER TABLE ONLY public.combined_tbl\n    ADD CONSTRAINT combined_tbl_bittv_key UNIQUE (bittv);"
 				},
 				{
-					"ObjectName": "public.combined_tbl, constraint: uk",
+					"ObjectName": "public.combined_tbl, constraint: (uk)",
 					"SqlStatement": "ALTER TABLE ONLY public.combined_tbl\n    ADD CONSTRAINT uk UNIQUE (lsn);"
 				},
 				{
-					"ObjectName": "public.combined_tbl, constraint: combined_tbl_pkey",
+					"ObjectName": "public.combined_tbl, constraint: (combined_tbl_pkey)",
 					"SqlStatement": "ALTER TABLE ONLY public.combined_tbl\n    ADD CONSTRAINT combined_tbl_pkey PRIMARY KEY (id, arr_enum);"
 				}
 			],
@@ -551,7 +551,40 @@
 			],
 			"DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unlogged-table-is-not-supported",
 			"MinimumVersionsFixedIn": null
-		}
+		},
+		{
+			"FeatureName": "System Columns",
+			"Objects": [
+				{
+					"ObjectName": "public.ordersentry_view",
+					"SqlStatement": "CREATE VIEW public.ordersentry_view AS\n SELECT ordersentry.order_id,\n    ordersentry.customer_name,\n    ordersentry.product_name,\n    ordersentry.quantity,\n    ordersentry.price,\n    XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name), XMLELEMENT(NAME \"Quantity\", ordersentry.quantity), XMLELEMENT(NAME \"TotalPrice\", (ordersentry.price * (ordersentry.quantity)::numeric))) AS order_xml,\n    XMLCONCAT(XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name)) AS summary_xml,\n    pg_try_advisory_lock((hashtext((ordersentry.customer_name || ordersentry.product_name)))::bigint) AS lock_acquired,\n    ordersentry.ctid AS row_ctid,\n    ordersentry.xmin AS transaction_id\n   FROM public.ordersentry;"
+				}
+			],
+			"DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported",
+			"MinimumVersionsFixedIn": null
+		},
+		{
+			"FeatureName": "XML Functions",
+			"Objects": [
+				{
+					"ObjectName": "public.ordersentry_view",
+					"SqlStatement": "CREATE VIEW public.ordersentry_view AS\n SELECT ordersentry.order_id,\n    ordersentry.customer_name,\n    ordersentry.product_name,\n    ordersentry.quantity,\n    ordersentry.price,\n    XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name), XMLELEMENT(NAME \"Quantity\", ordersentry.quantity), XMLELEMENT(NAME \"TotalPrice\", (ordersentry.price * (ordersentry.quantity)::numeric))) AS order_xml,\n    XMLCONCAT(XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name)) AS summary_xml,\n    pg_try_advisory_lock((hashtext((ordersentry.customer_name || ordersentry.product_name)))::bigint) AS lock_acquired,\n    ordersentry.ctid AS row_ctid,\n    ordersentry.xmin AS transaction_id\n   FROM public.ordersentry;"
+				}
+			],
+			"DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported",
+		    "MinimumVersionsFixedIn": null
+        },
+		{
+			"FeatureName": "Advisory Locks",
+			"Objects": [
+				{
+					"ObjectName": "public.ordersentry_view",
+					"SqlStatement": "CREATE VIEW public.ordersentry_view AS\n SELECT ordersentry.order_id,\n    ordersentry.customer_name,\n    ordersentry.product_name,\n    ordersentry.quantity,\n    ordersentry.price,\n    XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name), XMLELEMENT(NAME \"Quantity\", ordersentry.quantity), XMLELEMENT(NAME \"TotalPrice\", (ordersentry.price * (ordersentry.quantity)::numeric))) AS order_xml,\n    XMLCONCAT(XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name)) AS summary_xml,\n    pg_try_advisory_lock((hashtext((ordersentry.customer_name || ordersentry.product_name)))::bigint) AS lock_acquired,\n    ordersentry.ctid AS row_ctid,\n    ordersentry.xmin AS transaction_id\n   FROM public.ordersentry;"
+				}
+			],
+			"DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented",
+		    "MinimumVersionsFixedIn": null
+        }
 	],
 	"UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.",
 	"TableIndexStats": [
@@ -2189,39 +2222,10 @@
 					"ObjectType": "FUNCTION",
 					"ObjectName": "schema2.process_order",
 					"SqlStatement": "SELECT pg_advisory_unlock(orderid);"
-				},
-				{
-					"ObjectType": "VIEW",
-					"ObjectName": "public.ordersentry_view",
-					"SqlStatement": "SELECT ordersentry.order_id, ordersentry.customer_name, ordersentry.product_name, ordersentry.quantity, ordersentry.price, xmlelement(name \"OrderDetails\", xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name), xmlelement(name \"Quantity\", ordersentry.quantity), xmlelement(name \"TotalPrice\", ordersentry.price * ordersentry.quantity::numeric)) AS order_xml, xmlconcat(xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name)) AS summary_xml, pg_try_advisory_lock(hashtext(ordersentry.customer_name || ordersentry.product_name)::bigint) AS lock_acquired, ordersentry.ctid AS row_ctid, ordersentry.xmin AS transaction_id FROM public.ordersentry"
 				}
 			],
 			"DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented",
 		    "MinimumVersionsFixedIn": null
-        },
-		{
-			"FeatureName": "XML Functions",
-			"Objects": [
-				{
-					"ObjectType": "VIEW",
-					"ObjectName": "public.ordersentry_view",
-					"SqlStatement": "SELECT ordersentry.order_id, ordersentry.customer_name, ordersentry.product_name, ordersentry.quantity, ordersentry.price, xmlelement(name \"OrderDetails\", xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name), xmlelement(name \"Quantity\", ordersentry.quantity), xmlelement(name \"TotalPrice\", ordersentry.price * ordersentry.quantity::numeric)) AS order_xml, xmlconcat(xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name)) AS summary_xml, pg_try_advisory_lock(hashtext(ordersentry.customer_name || ordersentry.product_name)::bigint) AS lock_acquired, ordersentry.ctid AS row_ctid, ordersentry.xmin AS transaction_id FROM public.ordersentry"
-				}
-			],
-			"DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported",
-		    "MinimumVersionsFixedIn": null
-        },
-		{
-			"FeatureName": "System Columns",
-			"Objects": [
-				{
-					"ObjectType": "VIEW",
-					"ObjectName": "public.ordersentry_view",
-					"SqlStatement": "SELECT ordersentry.order_id, ordersentry.customer_name, ordersentry.product_name, ordersentry.quantity, ordersentry.price, xmlelement(name \"OrderDetails\", xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name), xmlelement(name \"Quantity\", ordersentry.quantity), xmlelement(name \"TotalPrice\", ordersentry.price * ordersentry.quantity::numeric)) AS order_xml, xmlconcat(xmlelement(name \"Customer\", ordersentry.customer_name), xmlelement(name \"Product\", ordersentry.product_name)) AS summary_xml, pg_try_advisory_lock(hashtext(ordersentry.customer_name || ordersentry.product_name)::bigint) AS lock_acquired, ordersentry.ctid AS row_ctid, ordersentry.xmin AS transaction_id FROM public.ordersentry"
-				}
-			],
-			"DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported",
-			"MinimumVersionsFixedIn": null
-		}
+        }
 	]
 }
diff --git a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json
@@ -248,6 +248,21 @@
 	"UnsupportedDataTypes": null,
 	"UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.",
 	"UnsupportedFeatures": [
+        {
+			"FeatureName": "Advisory Locks",
+			"Objects":[],
+			"MinimumVersionsFixedIn":null
+		},
+		{
+			"FeatureName":"System Columns",
+			"Objects":[],
+			"MinimumVersionsFixedIn":null
+		},
+		{
+			"FeatureName": "XML Functions",
+			"Objects": [],
+"MinimumVersionsFixedIn":null
+		},
 		{
 			"FeatureName": "GIST indexes",
 			"Objects": [],

diff --git a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json
@@ -493,6 +493,21 @@
 	"UnsupportedDataTypes": null,
 	"UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.",
 	"UnsupportedFeatures": [
+        {
+			"FeatureName": "Advisory Locks",
+			"Objects":[],
+			"MinimumVersionsFixedIn":null
+		},
+		{
+			"FeatureName":"System Columns",
+			"Objects":[],
+			"MinimumVersionsFixedIn":null
+		},
+		{
+			"FeatureName": "XML Functions",
+			"Objects": [],
+"MinimumVersionsFixedIn":null
+		},
 		{
 			"FeatureName": "GIST indexes",
 			"Objects": [

diff --git a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json
@@ -73,6 +73,21 @@
 	],
 	"UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.",
 	"UnsupportedFeatures": [
+        {
+			"FeatureName": "Advisory Locks",
+			"Objects":[],
+			"MinimumVersionsFixedIn":null
+		},
+		{
+			"FeatureName":"System Columns",
+			"Objects":[],
+			"MinimumVersionsFixedIn":null
+		},
+		{
+			"FeatureName": "XML Functions",
+			"Objects": [],
+"MinimumVersionsFixedIn":null
+		},
 		{
 			"FeatureName": "GIST indexes",
 			"Objects": [

diff --git a/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json
@@ -80,6 +80,21 @@
 	"UnsupportedDataTypes": null,
 	"UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.",
 	"UnsupportedFeatures": [
+        {
+			"FeatureName": "Advisory Locks",
+			"Objects":[],
+			"MinimumVersionsFixedIn":null
+		},
+		{
+			"FeatureName":"System Columns",
+			"Objects":[],
+			"MinimumVersionsFixedIn":null
+		},
+		{
+			"FeatureName": "XML Functions",
+			"Objects": [],
+"MinimumVersionsFixedIn":null
+		},
 		{
 			"FeatureName": "GIST indexes",
 			"Objects": [],

diff --git a/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json
@@ -377,6 +377,21 @@
 	"UnsupportedDataTypes": null,
 	"UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.",
 	"UnsupportedFeatures": [
+        {
+			"FeatureName": "Advisory Locks",
+			"Objects":[],
+			"MinimumVersionsFixedIn":null
+		},
+		{
+			"FeatureName":"System Columns",
+			"Objects":[],
+			"MinimumVersionsFixedIn":null
+		},
+		{
+			"FeatureName": "XML Functions",
+			"Objects": [],
+"MinimumVersionsFixedIn":null
+		},
 		{
 			"FeatureName": "GIST indexes",
 			"Objects": [],

diff --git a/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json
@@ -118,6 +118,21 @@
 	"UnsupportedDataTypes": null,
 	"UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.",
 	"UnsupportedFeatures": [
+        {
+			"FeatureName": "Advisory Locks",
+			"Objects":[],
+			"MinimumVersionsFixedIn":null
+		},
+		{
+			"FeatureName":"System Columns",
+			"Objects":[],
+			"MinimumVersionsFixedIn":null
+		},
+		{
+			"FeatureName": "XML Functions",
+			"Objects": [],
+"MinimumVersionsFixedIn":null
+		},
 		{
 			"FeatureName": "GIST indexes",
 			"Objects": [

diff --git a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json
@@ -76,6 +76,21 @@
 	"UnsupportedDataTypes": null,
 	"UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.",
 	"UnsupportedFeatures": [
+        {
+			"FeatureName": "Advisory Locks",
+			"Objects":[],
+			"MinimumVersionsFixedIn":null
+		},
+		{
+			"FeatureName":"System Columns",
+			"Objects":[],
+			"MinimumVersionsFixedIn":null
+		},
+		{
+			"FeatureName": "XML Functions",
+			"Objects": [],
+"MinimumVersionsFixedIn":null
+		},
 		{
 			"FeatureName": "GIST indexes",
 			"Objects": [],

diff --git a/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json
@@ -82,6 +82,21 @@
 	"UnsupportedDataTypes": null,
 	"UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.",
 	"UnsupportedFeatures": [
+        {
+			"FeatureName": "Advisory Locks",
+			"Objects":[],
+			"MinimumVersionsFixedIn":null
+		},
+		{
+			"FeatureName":"System Columns",
+			"Objects":[],
+			"MinimumVersionsFixedIn":null
+		},
+		{
+			"FeatureName": "XML Functions",
+			"Objects": [],
+"MinimumVersionsFixedIn":null
+		},
 		{
 			"FeatureName": "GIST indexes",
 			"Objects": [],

diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go
@@ -951,6 +951,9 @@ func getUnsupportedFeaturesFromSchemaAnalysisReport(featureName string, issueRea
 	var minVersionsFixedInSet bool
 
 	for _, issue := range schemaAnalysisReport.Issues {
+		if !slices.Contains([]string{UNSUPPORTED_FEATURES, MIGRATION_CAVEATS}, issue.IssueType) {
+			continue
+		}
 		if strings.Contains(issue.Reason, issueReason) {
 			objectInfo := ObjectInfo{
 				ObjectName:   issue.ObjectName,
@@ -1000,6 +1003,9 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem
 	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(UNLOGGED_TABLE_FEATURE, ISSUE_UNLOGGED_TABLE, schemaAnalysisReport, false, ""))
 	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REFERENCING_TRIGGER_FEATURE, REFERENCING_CLAUSE_FOR_TRIGGERS, schemaAnalysisReport, false, ""))
 	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE_FEATURE, BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE, schemaAnalysisReport, false, ""))
+	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport("Advisory Locks", "Advisory Locks", schemaAnalysisReport, false, ""))
+	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport("XML Functions", "XML Functions", schemaAnalysisReport, false, ""))
+	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport("System Columns", "System Columns", schemaAnalysisReport, false, ""))
 
 	return unsupportedFeatures, nil
 }