Renamed count headings in schema summary of html/txt report and fixed…

… all cases to add to invalid count (#2073) 1.Changes in the Analyze schema HTML/TXT reports and Assess-migration HTML report for Total Count -> Total Objects Valid Count -> Object Without Issues Invalid Count -> Object With Issues 2. Fixed all the reporting issue cases to add to the invalid count based on object names 3. Fixed all the test cases expected files for this
yugabyte · Dec 18, 2024 · 736fcee · 736fcee
1 parent 1f69c1c
commit 736fcee
Show file tree

Hide file tree

Showing 18 changed files with 168 additions and 133 deletions.
diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql
@@ -16,7 +16,7 @@ CREATE OR REPLACE view test AS (
                             FROM test1
                             where t = '1DAY' group by x
                             );
-
+CREATE VIEW view_name AS SELECT * from test_arr_enum;
 --Unsupported PG Syntax
 --For this case we will have two issues reported one by regex and other by Unsupported PG syntax with error msg
 ALTER VIEW view_name TO select * from test;

diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json
@@ -371,16 +371,6 @@
             "GH": "https://github.com/YugaByte/yugabyte-db/issues/1131",
             "MinimumVersionsFixedIn": null
         },
-        {
-            "IssueType": "unsupported_features",
-            "ObjectType": "VIEW",
-            "ObjectName": "",
-            "Reason": "Unsupported PG syntax - 'syntax error at or near \"TO\"'",
-            "SqlStatement": "ALTER VIEW view_name TO select * from test;",
-            "Suggestion": "Fix the schema as per PG syntax",
-            "GH": "https://github.com/yugabyte/yb-voyager/issues/1625",
-            "MinimumVersionsFixedIn": null
-        },
         {
             "IssueType": "unsupported_features",
             "ObjectType": "TABLE",

diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json
@@ -8,6 +8,7 @@
             {
                 "ObjectType": "SCHEMA",
                 "TotalCount": 1,
+                "InvalidCount": 1,
                 "ObjectNames": "hollywood"
             },
             {
@@ -20,17 +21,18 @@
             {
                 "ObjectType": "TYPE",
                 "TotalCount": 4,
+                "InvalidCount":1,
                 "ObjectNames": "address_type, non_public.address_type1, non_public.enum_test, enum_test"
             },
             {
                 "ObjectType": "TABLE",
                 "TotalCount": 50,
-                "InvalidCount": 33,
+                "InvalidCount": 42,
                 "ObjectNames": "public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr"            },
             {
                 "ObjectType": "INDEX",
                 "TotalCount": 43,
-                "InvalidCount": 38,
+                "InvalidCount": 39,
                 "ObjectNames": "idx1 ON combined_tbl1, idx2 ON combined_tbl1, idx3 ON combined_tbl1, idx4 ON combined_tbl1, idx5 ON combined_tbl1, idx6 ON combined_tbl1, idx7 ON combined_tbl1, idx8 ON combined_tbl1, film_fulltext_idx ON public.film, idx_actor_last_name ON public.actor, idx_name1 ON table_name, idx_name2 ON table_name, idx_name3 ON schema_name.table_name, idx_fileinfo_name_splitted ON public.fileinfo, abc ON public.example, abc ON schema2.example, tsvector_idx ON public.documents, tsquery_idx ON public.ts_query_table, idx_citext ON public.citext_type, idx_inet ON public.inet_type, idx_json ON public.test_jsonb, idx_json2 ON public.test_jsonb, idx_valid ON public.test_jsonb, idx_array ON public.documents, idx1 ON combined_tbl, idx2 ON combined_tbl, idx3 ON combined_tbl, idx4 ON combined_tbl, idx5 ON combined_tbl, idx6 ON combined_tbl, idx7 ON combined_tbl, idx8 ON combined_tbl, idx9 ON combined_tbl, idx10 ON combined_tbl, idx11 ON combined_tbl, idx12 ON combined_tbl, idx13 ON combined_tbl, idx14 ON combined_tbl, idx15 ON combined_tbl, idx_udt ON test_udt, idx_udt1 ON test_udt, idx_enum ON test_udt, \"idx\u0026_enum2\" ON test_udt",
                 "Details": "There are some GIN indexes present in the schema, but GIN indexes are partially supported in YugabyteDB as mentioned in (https://github.com/yugabyte/yugabyte-db/issues/7850) so take a look and modify them if not supported."
             },
@@ -48,9 +50,9 @@
             },
             {
                 "ObjectType": "VIEW",
-                "TotalCount": 4,
-                "InvalidCount": 4,
-                "ObjectNames": "v1, v2, test, public.orders_view"
+                "TotalCount": 5,
+                "InvalidCount": 5,
+                "ObjectNames": "v1, v2, test, public.orders_view, view_name"
             },
             {
                 "ObjectType": "TRIGGER",
@@ -67,7 +69,7 @@
             {
                 "ObjectType": "CONVERSION",
                 "TotalCount": 1,
-                "InvalidCount": 1,
+                "InvalidCount": 0,
                 "ObjectNames": "myconv"
             },
             {
@@ -85,6 +87,7 @@
             {
                 "ObjectType": "OPERATOR",
                 "TotalCount": 1,
+                "InvalidCount": 0,
                 "ObjectNames": "\u003c%"
             }
         ]

diff --git a/migtests/tests/analyze-schema/validate b/migtests/tests/analyze-schema/validate
@@ -45,25 +45,27 @@ def validate_report_summary(report, expected_summary):
 			validate_database_objects_summary(report, expected_summary)
 
 def validate_database_objects_summary(report, expected_summary):
-    key = "DatabaseObjects"
-    expected_objects = expected_summary.get(key, [])
-    reported_objects = report['Summary'].get(key, [])
+	key = "DatabaseObjects"
+	expected_objects = expected_summary.get(key, [])
+	reported_objects = report['Summary'].get(key, [])
 
-    assert len(expected_objects) == len(reported_objects), "Number of database objects does not match"
+	assert len(expected_objects) == len(reported_objects), "Number of database objects does not match"
 
-    for expected_obj, reported_obj in zip(expected_objects, reported_objects):
-        print(f"validating database object: {expected_obj['ObjectType']}")
-        print(f"expected summary field for {key}: {expected_obj}")
-        print(f"reported summary field for {key}: {reported_obj}")
-        assert expected_obj["ObjectType"] == reported_obj["ObjectType"], f"Object type mismatch for {expected_obj['ObjectType']}"
-        assert expected_obj["TotalCount"] == reported_obj["TotalCount"], f"Total count mismatch for {expected_obj['ObjectType']}"
 
-        if "Details" in expected_obj and "Details" in reported_obj:
-            assert expected_obj["Details"] == reported_obj["Details"], f"Details mismatch for {expected_obj['ObjectType']}"
+	for expected_obj, reported_obj in zip(expected_objects, reported_objects):
+		print(f"validating database object: {expected_obj['ObjectType']}")
+		print(f"expected summary field for {key}: {expected_obj}")
+		print(f"reported summary field for {key}: {reported_obj}")
+		assert expected_obj["InvalidCount"] == reported_obj["InvalidCount"], f"Invalid count mismatch for {expected_obj['ObjectType']}"
+		assert expected_obj["ObjectType"] == reported_obj["ObjectType"], f"Object type mismatch for {expected_obj['ObjectType']}"
+		assert expected_obj["TotalCount"] == reported_obj["TotalCount"], f"Total count mismatch for {expected_obj['ObjectType']}"
 
-        expected_names = sorted(expected_obj.get("ObjectNames", "").split(", "))
-        reported_names = sorted(reported_obj.get("ObjectNames", "").split(", "))
-        assert expected_names == reported_names, f"Object names mismatch for {expected_obj['ObjectType']}"
+		if "Details" in expected_obj and "Details" in reported_obj:
+			assert expected_obj["Details"] == reported_obj["Details"], f"Details mismatch for {expected_obj['ObjectType']}"
+
+		expected_names = sorted(expected_obj.get("ObjectNames", "").split(", "))
+		reported_names = sorted(reported_obj.get("ObjectNames", "").split(", "))
+		assert expected_names == reported_names, f"Object names mismatch for {expected_obj['ObjectType']}"
 
 def validate_report_issues(report, expected_issues):
     # FilePath reported in the report can be different depending on the machine

diff --git a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json
@@ -46,7 +46,7 @@
 			{
 				"ObjectType": "TABLE",
 				"TotalCount": 68,
-				"InvalidCount": 5,
+				"InvalidCount": 68,
 				"ObjectNames": "humanresources.department, humanresources.employee, humanresources.employeedepartmenthistory, humanresources.employeepayhistory, humanresources.jobcandidate, humanresources.shift, person.address, person.businessentityaddress, person.countryregion, person.emailaddress, person.person, person.personphone, person.phonenumbertype, person.stateprovince, person.addresstype, person.businessentity, person.businessentitycontact, person.contacttype, person.password, production.billofmaterials, production.culture, production.document, production.illustration, production.location, production.product, production.productcategory, production.productcosthistory, production.productdescription, production.productdocument, production.productinventory, production.productlistpricehistory, production.productmodel, production.productmodelillustration, production.productmodelproductdescriptionculture, production.productphoto, production.productproductphoto, production.productreview, production.productsubcategory, production.scrapreason, production.transactionhistory, production.transactionhistoryarchive, production.unitmeasure, production.workorder, production.workorderrouting, purchasing.purchaseorderdetail, purchasing.purchaseorderheader, purchasing.productvendor, purchasing.shipmethod, purchasing.vendor, sales.customer, sales.creditcard, sales.currencyrate, sales.countryregioncurrency, sales.currency, sales.personcreditcard, sales.store, sales.shoppingcartitem, sales.specialoffer, sales.salesorderdetail, sales.salesorderheader, sales.salesorderheadersalesreason, sales.specialofferproduct, sales.salesperson, sales.salespersonquotahistory, sales.salesreason, sales.salesterritory, sales.salesterritoryhistory, sales.salestaxrate"
 			},
 			{

diff --git a/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json
@@ -46,8 +46,7 @@
             {
                 "ObjectType": "TABLE",
                 "TotalCount": 68,
-                "InvalidCount": 0,
-                "InvalidCount": 5,
+                "InvalidCount": 68,
                 "ObjectNames": "humanresources.department, humanresources.employee, humanresources.employeedepartmenthistory, humanresources.employeepayhistory, humanresources.jobcandidate, humanresources.shift, person.address, person.businessentityaddress, person.countryregion, person.emailaddress, person.person, person.personphone, person.phonenumbertype, person.stateprovince, person.addresstype, person.businessentity, person.businessentitycontact, person.contacttype, person.password, production.billofmaterials, production.culture, production.document, production.illustration, production.location, production.product, production.productcategory, production.productcosthistory, production.productdescription, production.productdocument, production.productinventory, production.productlistpricehistory, production.productmodel, production.productmodelillustration, production.productmodelproductdescriptionculture, production.productphoto, production.productproductphoto, production.productreview, production.productsubcategory, production.scrapreason, production.transactionhistory, production.transactionhistoryarchive, production.unitmeasure, production.workorder, production.workorderrouting, purchasing.purchaseorderdetail, purchasing.purchaseorderheader, purchasing.productvendor, purchasing.shipmethod, purchasing.vendor, sales.customer, sales.creditcard, sales.currencyrate, sales.countryregioncurrency, sales.currency, sales.personcreditcard, sales.store, sales.shoppingcartitem, sales.specialoffer, sales.salesorderdetail, sales.salesorderheader, sales.salesorderheadersalesreason, sales.specialofferproduct, sales.salesperson, sales.salespersonquotahistory, sales.salesreason, sales.salesterritory, sales.salesterritoryhistory, sales.salestaxrate"
             },
             {

diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json
@@ -80,7 +80,7 @@
 			{
 				"ObjectType": "TRIGGER",
 				"TotalCount": 3,
-				"InvalidCount": 1,
+				"InvalidCount": 3,
 				"ObjectNames": "audit_trigger ON public.tt, before_sales_region_insert_update ON public.sales_region, audit_trigger ON schema2.tt"
 			},
 			{

diff --git a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json
@@ -24,7 +24,7 @@
 			{
 				"ObjectType": "TABLE",
 				"TotalCount": 175,
-				"InvalidCount": 0,
+				"InvalidCount": 62,
 				"ObjectNames": "mgd.acc_accession, mgd.acc_accessionmax, mgd.acc_accessionreference, mgd.acc_actualdb, mgd.acc_logicaldb, mgd.acc_mgitype, mgd.all_allele, mgd.all_allele_cellline, mgd.all_cellline, mgd.all_cellline_derivation, mgd.mgi_user, mgd.prb_strain, mgd.voc_term, mgd.mgi_organism, mgd.mgi_relationship, mgd.mrk_marker, mgd.all_allele_mutation, mgd.voc_annot, mgd.bib_citation_cache, mgd.gxd_allelepair, mgd.voc_annottype, mgd.all_cre_cache, mgd.all_knockout_cache, mgd.all_label, mgd.gxd_allelegenotype, mgd.mgi_reference_assoc, mgd.all_variant, mgd.all_variant_sequence, mgd.bib_refs, mgd.gxd_expression, mgd.gxd_index, mgd.img_image, mgd.mgi_refassoctype, mgd.mgi_synonym, mgd.mgi_synonymtype, mgd.mld_expts, mgd.mld_notes, mgd.mrk_do_cache, mgd.mrk_reference, mgd.mrk_strainmarker, mgd.prb_reference, mgd.prb_source, mgd.voc_evidence, mgd.bib_books, mgd.bib_workflow_status, mgd.bib_notes, mgd.gxd_assay, mgd.gxd_specimen, mgd.bib_workflow_data, mgd.bib_workflow_relevance, mgd.bib_workflow_tag, mgd.crs_cross, mgd.crs_matrix, mgd.crs_progeny, mgd.crs_references, mgd.crs_typings, mgd.dag_closure, mgd.dag_dag, mgd.dag_edge, mgd.dag_label, mgd.dag_node, mgd.voc_vocabdag, mgd.go_tracking, mgd.gxd_antibody, mgd.gxd_antigen, mgd.gxd_antibodyalias, mgd.gxd_antibodymarker, mgd.gxd_antibodyprep, mgd.gxd_gellane, mgd.gxd_insituresult, mgd.gxd_insituresultimage, mgd.gxd_assaytype, mgd.gxd_assaynote, mgd.gxd_gelband, mgd.gxd_gelrow, mgd.gxd_genotype, mgd.gxd_gellanestructure, mgd.gxd_theilerstage, mgd.voc_annotheader, mgd.gxd_htexperiment, mgd.gxd_htexperimentvariable, mgd.gxd_htrawsample, mgd.gxd_htsample, mgd.gxd_htsample_rnaseq, mgd.gxd_htsample_rnaseqcombined, mgd.gxd_htsample_rnaseqset, mgd.gxd_htsample_rnaseqset_cache, mgd.gxd_htsample_rnaseqsetmember, mgd.gxd_index_stages, mgd.gxd_isresultcelltype, mgd.img_imagepane, mgd.gxd_isresultstructure, mgd.gxd_probeprep, mgd.prb_probe, mgd.img_imagepane_assoc, mgd.mgi_note, mgd.map_coord_collection, mgd.map_coord_feature, mgd.map_coordinate, mgd.mrk_chromosome, mgd.mgi_dbinfo, mgd.mgi_keyvalue, mgd.mgi_notetype, mgd.mgi_organism_mgitype, mgd.mgi_property, mgd.mgi_propertytype, mgd.mgi_relationship_category, mgd.mgi_relationship_property, mgd.mgi_set, mgd.mgi_setmember, mgd.mgi_setmember_emapa, mgd.mgi_translation, mgd.mgi_translationtype, mgd.voc_vocab, mgd.mld_assay_types, mgd.mld_concordance, mgd.mld_contig, mgd.mld_contigprobe, mgd.mld_expt_marker, mgd.mld_expt_notes, mgd.mld_fish, mgd.mld_fish_region, mgd.mld_hit, mgd.mld_hybrid, mgd.mld_insitu, mgd.mld_isregion, mgd.mld_matrix, mgd.mld_mc2point, mgd.mld_mcdatalist, mgd.mld_ri, mgd.mld_ri2point, mgd.mld_ridata, mgd.mld_statistics, mgd.mrk_types, mgd.mrk_biotypemapping, mgd.mrk_cluster, mgd.mrk_clustermember, mgd.mrk_current, mgd.mrk_history, mgd.mrk_label, mgd.mrk_location_cache, mgd.mrk_status, mgd.mrk_mcv_cache, mgd.mrk_mcv_count_cache, mgd.mrk_notes, mgd.prb_alias, mgd.prb_allele, mgd.prb_allele_strain, mgd.prb_marker, mgd.prb_notes, mgd.prb_ref_notes, mgd.prb_rflv, mgd.prb_tissue, mgd.prb_strain_genotype, mgd.prb_strain_marker, mgd.ri_riset, mgd.ri_summary, mgd.ri_summary_expt_ref, mgd.seq_allele_assoc, mgd.seq_coord_cache, mgd.seq_genemodel, mgd.seq_genetrap, mgd.seq_marker_cache, mgd.seq_probe_cache, mgd.seq_sequence, mgd.seq_sequence_assoc, mgd.seq_sequence_raw, mgd.seq_source_assoc, mgd.voc_allele_cache, mgd.voc_annot_count_cache, mgd.voc_evidence_property, mgd.voc_marker_cache, mgd.voc_term_emapa, mgd.voc_term_emaps, mgd.wks_rosetta"
 			},
 			{

diff --git a/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json
@@ -24,7 +24,7 @@
             {
                 "ObjectType": "TABLE",
                 "TotalCount": 175,
-                "InvalidCount": 0,
+                "InvalidCount": 62,
                 "ObjectNames": "mgd.acc_accession, mgd.acc_accessionmax, mgd.acc_accessionreference, mgd.acc_actualdb, mgd.acc_logicaldb, mgd.acc_mgitype, mgd.all_allele, mgd.all_allele_cellline, mgd.all_cellline, mgd.all_cellline_derivation, mgd.mgi_user, mgd.prb_strain, mgd.voc_term, mgd.mgi_organism, mgd.mgi_relationship, mgd.mrk_marker, mgd.all_allele_mutation, mgd.voc_annot, mgd.bib_citation_cache, mgd.gxd_allelepair, mgd.voc_annottype, mgd.all_cre_cache, mgd.all_knockout_cache, mgd.all_label, mgd.gxd_allelegenotype, mgd.mgi_reference_assoc, mgd.all_variant, mgd.all_variant_sequence, mgd.bib_refs, mgd.gxd_expression, mgd.gxd_index, mgd.img_image, mgd.mgi_refassoctype, mgd.mgi_synonym, mgd.mgi_synonymtype, mgd.mld_expts, mgd.mld_notes, mgd.mrk_do_cache, mgd.mrk_reference, mgd.mrk_strainmarker, mgd.prb_reference, mgd.prb_source, mgd.voc_evidence, mgd.bib_books, mgd.bib_workflow_status, mgd.bib_notes, mgd.gxd_assay, mgd.gxd_specimen, mgd.bib_workflow_data, mgd.bib_workflow_relevance, mgd.bib_workflow_tag, mgd.crs_cross, mgd.crs_matrix, mgd.crs_progeny, mgd.crs_references, mgd.crs_typings, mgd.dag_closure, mgd.dag_dag, mgd.dag_edge, mgd.dag_label, mgd.dag_node, mgd.voc_vocabdag, mgd.go_tracking, mgd.gxd_antibody, mgd.gxd_antigen, mgd.gxd_antibodyalias, mgd.gxd_antibodymarker, mgd.gxd_antibodyprep, mgd.gxd_gellane, mgd.gxd_insituresult, mgd.gxd_insituresultimage, mgd.gxd_assaytype, mgd.gxd_assaynote, mgd.gxd_gelband, mgd.gxd_gelrow, mgd.gxd_genotype, mgd.gxd_gellanestructure, mgd.gxd_theilerstage, mgd.voc_annotheader, mgd.gxd_htexperiment, mgd.gxd_htexperimentvariable, mgd.gxd_htrawsample, mgd.gxd_htsample, mgd.gxd_htsample_rnaseq, mgd.gxd_htsample_rnaseqcombined, mgd.gxd_htsample_rnaseqset, mgd.gxd_htsample_rnaseqset_cache, mgd.gxd_htsample_rnaseqsetmember, mgd.gxd_index_stages, mgd.gxd_isresultcelltype, mgd.img_imagepane, mgd.gxd_isresultstructure, mgd.gxd_probeprep, mgd.prb_probe, mgd.img_imagepane_assoc, mgd.mgi_note, mgd.map_coord_collection, mgd.map_coord_feature, mgd.map_coordinate, mgd.mrk_chromosome, mgd.mgi_dbinfo, mgd.mgi_keyvalue, mgd.mgi_notetype, mgd.mgi_organism_mgitype, mgd.mgi_property, mgd.mgi_propertytype, mgd.mgi_relationship_category, mgd.mgi_relationship_property, mgd.mgi_set, mgd.mgi_setmember, mgd.mgi_setmember_emapa, mgd.mgi_translation, mgd.mgi_translationtype, mgd.voc_vocab, mgd.mld_assay_types, mgd.mld_concordance, mgd.mld_contig, mgd.mld_contigprobe, mgd.mld_expt_marker, mgd.mld_expt_notes, mgd.mld_fish, mgd.mld_fish_region, mgd.mld_hit, mgd.mld_hybrid, mgd.mld_insitu, mgd.mld_isregion, mgd.mld_matrix, mgd.mld_mc2point, mgd.mld_mcdatalist, mgd.mld_ri, mgd.mld_ri2point, mgd.mld_ridata, mgd.mld_statistics, mgd.mrk_types, mgd.mrk_biotypemapping, mgd.mrk_cluster, mgd.mrk_clustermember, mgd.mrk_current, mgd.mrk_history, mgd.mrk_label, mgd.mrk_location_cache, mgd.mrk_status, mgd.mrk_mcv_cache, mgd.mrk_mcv_count_cache, mgd.mrk_notes, mgd.prb_alias, mgd.prb_allele, mgd.prb_allele_strain, mgd.prb_marker, mgd.prb_notes, mgd.prb_ref_notes, mgd.prb_rflv, mgd.prb_tissue, mgd.prb_strain_genotype, mgd.prb_strain_marker, mgd.ri_riset, mgd.ri_summary, mgd.ri_summary_expt_ref, mgd.seq_allele_assoc, mgd.seq_coord_cache, mgd.seq_genemodel, mgd.seq_genetrap, mgd.seq_marker_cache, mgd.seq_probe_cache, mgd.seq_sequence, mgd.seq_sequence_assoc, mgd.seq_sequence_raw, mgd.seq_source_assoc, mgd.voc_allele_cache, mgd.voc_annot_count_cache, mgd.voc_evidence_property, mgd.voc_marker_cache, mgd.voc_term_emapa, mgd.voc_term_emaps, mgd.wks_rosetta"
             },
             {