diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql index d018df373f..435adb925f 100644 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql @@ -16,7 +16,7 @@ CREATE OR REPLACE view test AS ( FROM test1 where t = '1DAY' group by x ); - +CREATE VIEW view_name AS SELECT * from test_arr_enum; --Unsupported PG Syntax --For this case we will have two issues reported one by regex and other by Unsupported PG syntax with error msg ALTER VIEW view_name TO select * from test; diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index 6b44e2beb8..899d852562 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -371,16 +371,6 @@ "GH": "https://github.com/YugaByte/yugabyte-db/issues/1131", "MinimumVersionsFixedIn": null }, - { - "IssueType": "unsupported_features", - "ObjectType": "VIEW", - "ObjectName": "", - "Reason": "Unsupported PG syntax - 'syntax error at or near \"TO\"'", - "SqlStatement": "ALTER VIEW view_name TO select * from test;", - "Suggestion": "Fix the schema as per PG syntax", - "GH": "https://github.com/yugabyte/yb-voyager/issues/1625", - "MinimumVersionsFixedIn": null - }, { "IssueType": "unsupported_features", "ObjectType": "TABLE", diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index e047b08fd4..fa71b0969b 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -8,6 +8,7 @@ { "ObjectType": "SCHEMA", "TotalCount": 1, + "InvalidCount": 1, "ObjectNames": "hollywood" }, { @@ -20,17 +21,18 @@ { "ObjectType": "TYPE", "TotalCount": 4, + "InvalidCount":1, "ObjectNames": "address_type, non_public.address_type1, non_public.enum_test, enum_test" }, { "ObjectType": "TABLE", "TotalCount": 50, - "InvalidCount": 33, + "InvalidCount": 42, "ObjectNames": "public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr" }, { "ObjectType": "INDEX", "TotalCount": 43, - "InvalidCount": 38, + "InvalidCount": 39, "ObjectNames": "idx1 ON combined_tbl1, idx2 ON combined_tbl1, idx3 ON combined_tbl1, idx4 ON combined_tbl1, idx5 ON combined_tbl1, idx6 ON combined_tbl1, idx7 ON combined_tbl1, idx8 ON combined_tbl1, film_fulltext_idx ON public.film, idx_actor_last_name ON public.actor, idx_name1 ON table_name, idx_name2 ON table_name, idx_name3 ON schema_name.table_name, idx_fileinfo_name_splitted ON public.fileinfo, abc ON public.example, abc ON schema2.example, tsvector_idx ON public.documents, tsquery_idx ON public.ts_query_table, idx_citext ON public.citext_type, idx_inet ON public.inet_type, idx_json ON public.test_jsonb, idx_json2 ON public.test_jsonb, idx_valid ON public.test_jsonb, idx_array ON public.documents, idx1 ON combined_tbl, idx2 ON combined_tbl, idx3 ON combined_tbl, idx4 ON combined_tbl, idx5 ON combined_tbl, idx6 ON combined_tbl, idx7 ON combined_tbl, idx8 ON combined_tbl, idx9 ON combined_tbl, idx10 ON combined_tbl, idx11 ON combined_tbl, idx12 ON combined_tbl, idx13 ON combined_tbl, idx14 ON combined_tbl, idx15 ON combined_tbl, idx_udt ON test_udt, idx_udt1 ON test_udt, idx_enum ON test_udt, \"idx\u0026_enum2\" ON test_udt", "Details": "There are some GIN indexes present in the schema, but GIN indexes are partially supported in YugabyteDB as mentioned in (https://github.com/yugabyte/yugabyte-db/issues/7850) so take a look and modify them if not supported." }, @@ -48,9 +50,9 @@ }, { "ObjectType": "VIEW", - "TotalCount": 4, - "InvalidCount": 4, - "ObjectNames": "v1, v2, test, public.orders_view" + "TotalCount": 5, + "InvalidCount": 5, + "ObjectNames": "v1, v2, test, public.orders_view, view_name" }, { "ObjectType": "TRIGGER", @@ -67,7 +69,7 @@ { "ObjectType": "CONVERSION", "TotalCount": 1, - "InvalidCount": 1, + "InvalidCount": 0, "ObjectNames": "myconv" }, { @@ -85,6 +87,7 @@ { "ObjectType": "OPERATOR", "TotalCount": 1, + "InvalidCount": 0, "ObjectNames": "\u003c%" } ] diff --git a/migtests/tests/analyze-schema/validate b/migtests/tests/analyze-schema/validate index daf6faa023..64ee24da7d 100755 --- a/migtests/tests/analyze-schema/validate +++ b/migtests/tests/analyze-schema/validate @@ -45,25 +45,27 @@ def validate_report_summary(report, expected_summary): validate_database_objects_summary(report, expected_summary) def validate_database_objects_summary(report, expected_summary): - key = "DatabaseObjects" - expected_objects = expected_summary.get(key, []) - reported_objects = report['Summary'].get(key, []) + key = "DatabaseObjects" + expected_objects = expected_summary.get(key, []) + reported_objects = report['Summary'].get(key, []) - assert len(expected_objects) == len(reported_objects), "Number of database objects does not match" + assert len(expected_objects) == len(reported_objects), "Number of database objects does not match" - for expected_obj, reported_obj in zip(expected_objects, reported_objects): - print(f"validating database object: {expected_obj['ObjectType']}") - print(f"expected summary field for {key}: {expected_obj}") - print(f"reported summary field for {key}: {reported_obj}") - assert expected_obj["ObjectType"] == reported_obj["ObjectType"], f"Object type mismatch for {expected_obj['ObjectType']}" - assert expected_obj["TotalCount"] == reported_obj["TotalCount"], f"Total count mismatch for {expected_obj['ObjectType']}" - if "Details" in expected_obj and "Details" in reported_obj: - assert expected_obj["Details"] == reported_obj["Details"], f"Details mismatch for {expected_obj['ObjectType']}" + for expected_obj, reported_obj in zip(expected_objects, reported_objects): + print(f"validating database object: {expected_obj['ObjectType']}") + print(f"expected summary field for {key}: {expected_obj}") + print(f"reported summary field for {key}: {reported_obj}") + assert expected_obj["InvalidCount"] == reported_obj["InvalidCount"], f"Invalid count mismatch for {expected_obj['ObjectType']}" + assert expected_obj["ObjectType"] == reported_obj["ObjectType"], f"Object type mismatch for {expected_obj['ObjectType']}" + assert expected_obj["TotalCount"] == reported_obj["TotalCount"], f"Total count mismatch for {expected_obj['ObjectType']}" - expected_names = sorted(expected_obj.get("ObjectNames", "").split(", ")) - reported_names = sorted(reported_obj.get("ObjectNames", "").split(", ")) - assert expected_names == reported_names, f"Object names mismatch for {expected_obj['ObjectType']}" + if "Details" in expected_obj and "Details" in reported_obj: + assert expected_obj["Details"] == reported_obj["Details"], f"Details mismatch for {expected_obj['ObjectType']}" + + expected_names = sorted(expected_obj.get("ObjectNames", "").split(", ")) + reported_names = sorted(reported_obj.get("ObjectNames", "").split(", ")) + assert expected_names == reported_names, f"Object names mismatch for {expected_obj['ObjectType']}" def validate_report_issues(report, expected_issues): # FilePath reported in the report can be different depending on the machine diff --git a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json index ceecd2595a..1ed87328e6 100755 --- a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json @@ -46,7 +46,7 @@ { "ObjectType": "TABLE", "TotalCount": 68, - "InvalidCount": 5, + "InvalidCount": 68, "ObjectNames": "humanresources.department, humanresources.employee, humanresources.employeedepartmenthistory, humanresources.employeepayhistory, humanresources.jobcandidate, humanresources.shift, person.address, person.businessentityaddress, person.countryregion, person.emailaddress, person.person, person.personphone, person.phonenumbertype, person.stateprovince, person.addresstype, person.businessentity, person.businessentitycontact, person.contacttype, person.password, production.billofmaterials, production.culture, production.document, production.illustration, production.location, production.product, production.productcategory, production.productcosthistory, production.productdescription, production.productdocument, production.productinventory, production.productlistpricehistory, production.productmodel, production.productmodelillustration, production.productmodelproductdescriptionculture, production.productphoto, production.productproductphoto, production.productreview, production.productsubcategory, production.scrapreason, production.transactionhistory, production.transactionhistoryarchive, production.unitmeasure, production.workorder, production.workorderrouting, purchasing.purchaseorderdetail, purchasing.purchaseorderheader, purchasing.productvendor, purchasing.shipmethod, purchasing.vendor, sales.customer, sales.creditcard, sales.currencyrate, sales.countryregioncurrency, sales.currency, sales.personcreditcard, sales.store, sales.shoppingcartitem, sales.specialoffer, sales.salesorderdetail, sales.salesorderheader, sales.salesorderheadersalesreason, sales.specialofferproduct, sales.salesperson, sales.salespersonquotahistory, sales.salesreason, sales.salesterritory, sales.salesterritoryhistory, sales.salestaxrate" }, { diff --git a/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json index 728396eb1c..fa0a6e7955 100755 --- a/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json @@ -46,8 +46,7 @@ { "ObjectType": "TABLE", "TotalCount": 68, - "InvalidCount": 0, - "InvalidCount": 5, + "InvalidCount": 68, "ObjectNames": "humanresources.department, humanresources.employee, humanresources.employeedepartmenthistory, humanresources.employeepayhistory, humanresources.jobcandidate, humanresources.shift, person.address, person.businessentityaddress, person.countryregion, person.emailaddress, person.person, person.personphone, person.phonenumbertype, person.stateprovince, person.addresstype, person.businessentity, person.businessentitycontact, person.contacttype, person.password, production.billofmaterials, production.culture, production.document, production.illustration, production.location, production.product, production.productcategory, production.productcosthistory, production.productdescription, production.productdocument, production.productinventory, production.productlistpricehistory, production.productmodel, production.productmodelillustration, production.productmodelproductdescriptionculture, production.productphoto, production.productproductphoto, production.productreview, production.productsubcategory, production.scrapreason, production.transactionhistory, production.transactionhistoryarchive, production.unitmeasure, production.workorder, production.workorderrouting, purchasing.purchaseorderdetail, purchasing.purchaseorderheader, purchasing.productvendor, purchasing.shipmethod, purchasing.vendor, sales.customer, sales.creditcard, sales.currencyrate, sales.countryregioncurrency, sales.currency, sales.personcreditcard, sales.store, sales.shoppingcartitem, sales.specialoffer, sales.salesorderdetail, sales.salesorderheader, sales.salesorderheadersalesreason, sales.specialofferproduct, sales.salesperson, sales.salespersonquotahistory, sales.salesreason, sales.salesterritory, sales.salesterritoryhistory, sales.salestaxrate" }, { diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 6021709791..c5171c5d74 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -80,7 +80,7 @@ { "ObjectType": "TRIGGER", "TotalCount": 3, - "InvalidCount": 1, + "InvalidCount": 3, "ObjectNames": "audit_trigger ON public.tt, before_sales_region_insert_update ON public.sales_region, audit_trigger ON schema2.tt" }, { diff --git a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json index 77a3e8bdfe..bb9c5d74a8 100644 --- a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json @@ -24,7 +24,7 @@ { "ObjectType": "TABLE", "TotalCount": 175, - "InvalidCount": 0, + "InvalidCount": 62, "ObjectNames": "mgd.acc_accession, mgd.acc_accessionmax, mgd.acc_accessionreference, mgd.acc_actualdb, mgd.acc_logicaldb, mgd.acc_mgitype, mgd.all_allele, mgd.all_allele_cellline, mgd.all_cellline, mgd.all_cellline_derivation, mgd.mgi_user, mgd.prb_strain, mgd.voc_term, mgd.mgi_organism, mgd.mgi_relationship, mgd.mrk_marker, mgd.all_allele_mutation, mgd.voc_annot, mgd.bib_citation_cache, mgd.gxd_allelepair, mgd.voc_annottype, mgd.all_cre_cache, mgd.all_knockout_cache, mgd.all_label, mgd.gxd_allelegenotype, mgd.mgi_reference_assoc, mgd.all_variant, mgd.all_variant_sequence, mgd.bib_refs, mgd.gxd_expression, mgd.gxd_index, mgd.img_image, mgd.mgi_refassoctype, mgd.mgi_synonym, mgd.mgi_synonymtype, mgd.mld_expts, mgd.mld_notes, mgd.mrk_do_cache, mgd.mrk_reference, mgd.mrk_strainmarker, mgd.prb_reference, mgd.prb_source, mgd.voc_evidence, mgd.bib_books, mgd.bib_workflow_status, mgd.bib_notes, mgd.gxd_assay, mgd.gxd_specimen, mgd.bib_workflow_data, mgd.bib_workflow_relevance, mgd.bib_workflow_tag, mgd.crs_cross, mgd.crs_matrix, mgd.crs_progeny, mgd.crs_references, mgd.crs_typings, mgd.dag_closure, mgd.dag_dag, mgd.dag_edge, mgd.dag_label, mgd.dag_node, mgd.voc_vocabdag, mgd.go_tracking, mgd.gxd_antibody, mgd.gxd_antigen, mgd.gxd_antibodyalias, mgd.gxd_antibodymarker, mgd.gxd_antibodyprep, mgd.gxd_gellane, mgd.gxd_insituresult, mgd.gxd_insituresultimage, mgd.gxd_assaytype, mgd.gxd_assaynote, mgd.gxd_gelband, mgd.gxd_gelrow, mgd.gxd_genotype, mgd.gxd_gellanestructure, mgd.gxd_theilerstage, mgd.voc_annotheader, mgd.gxd_htexperiment, mgd.gxd_htexperimentvariable, mgd.gxd_htrawsample, mgd.gxd_htsample, mgd.gxd_htsample_rnaseq, mgd.gxd_htsample_rnaseqcombined, mgd.gxd_htsample_rnaseqset, mgd.gxd_htsample_rnaseqset_cache, mgd.gxd_htsample_rnaseqsetmember, mgd.gxd_index_stages, mgd.gxd_isresultcelltype, mgd.img_imagepane, mgd.gxd_isresultstructure, mgd.gxd_probeprep, mgd.prb_probe, mgd.img_imagepane_assoc, mgd.mgi_note, mgd.map_coord_collection, mgd.map_coord_feature, mgd.map_coordinate, mgd.mrk_chromosome, mgd.mgi_dbinfo, mgd.mgi_keyvalue, mgd.mgi_notetype, mgd.mgi_organism_mgitype, mgd.mgi_property, mgd.mgi_propertytype, mgd.mgi_relationship_category, mgd.mgi_relationship_property, mgd.mgi_set, mgd.mgi_setmember, mgd.mgi_setmember_emapa, mgd.mgi_translation, mgd.mgi_translationtype, mgd.voc_vocab, mgd.mld_assay_types, mgd.mld_concordance, mgd.mld_contig, mgd.mld_contigprobe, mgd.mld_expt_marker, mgd.mld_expt_notes, mgd.mld_fish, mgd.mld_fish_region, mgd.mld_hit, mgd.mld_hybrid, mgd.mld_insitu, mgd.mld_isregion, mgd.mld_matrix, mgd.mld_mc2point, mgd.mld_mcdatalist, mgd.mld_ri, mgd.mld_ri2point, mgd.mld_ridata, mgd.mld_statistics, mgd.mrk_types, mgd.mrk_biotypemapping, mgd.mrk_cluster, mgd.mrk_clustermember, mgd.mrk_current, mgd.mrk_history, mgd.mrk_label, mgd.mrk_location_cache, mgd.mrk_status, mgd.mrk_mcv_cache, mgd.mrk_mcv_count_cache, mgd.mrk_notes, mgd.prb_alias, mgd.prb_allele, mgd.prb_allele_strain, mgd.prb_marker, mgd.prb_notes, mgd.prb_ref_notes, mgd.prb_rflv, mgd.prb_tissue, mgd.prb_strain_genotype, mgd.prb_strain_marker, mgd.ri_riset, mgd.ri_summary, mgd.ri_summary_expt_ref, mgd.seq_allele_assoc, mgd.seq_coord_cache, mgd.seq_genemodel, mgd.seq_genetrap, mgd.seq_marker_cache, mgd.seq_probe_cache, mgd.seq_sequence, mgd.seq_sequence_assoc, mgd.seq_sequence_raw, mgd.seq_source_assoc, mgd.voc_allele_cache, mgd.voc_annot_count_cache, mgd.voc_evidence_property, mgd.voc_marker_cache, mgd.voc_term_emapa, mgd.voc_term_emaps, mgd.wks_rosetta" }, { diff --git a/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json index eb4e17c785..cfe0307c16 100644 --- a/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json @@ -24,7 +24,7 @@ { "ObjectType": "TABLE", "TotalCount": 175, - "InvalidCount": 0, + "InvalidCount": 62, "ObjectNames": "mgd.acc_accession, mgd.acc_accessionmax, mgd.acc_accessionreference, mgd.acc_actualdb, mgd.acc_logicaldb, mgd.acc_mgitype, mgd.all_allele, mgd.all_allele_cellline, mgd.all_cellline, mgd.all_cellline_derivation, mgd.mgi_user, mgd.prb_strain, mgd.voc_term, mgd.mgi_organism, mgd.mgi_relationship, mgd.mrk_marker, mgd.all_allele_mutation, mgd.voc_annot, mgd.bib_citation_cache, mgd.gxd_allelepair, mgd.voc_annottype, mgd.all_cre_cache, mgd.all_knockout_cache, mgd.all_label, mgd.gxd_allelegenotype, mgd.mgi_reference_assoc, mgd.all_variant, mgd.all_variant_sequence, mgd.bib_refs, mgd.gxd_expression, mgd.gxd_index, mgd.img_image, mgd.mgi_refassoctype, mgd.mgi_synonym, mgd.mgi_synonymtype, mgd.mld_expts, mgd.mld_notes, mgd.mrk_do_cache, mgd.mrk_reference, mgd.mrk_strainmarker, mgd.prb_reference, mgd.prb_source, mgd.voc_evidence, mgd.bib_books, mgd.bib_workflow_status, mgd.bib_notes, mgd.gxd_assay, mgd.gxd_specimen, mgd.bib_workflow_data, mgd.bib_workflow_relevance, mgd.bib_workflow_tag, mgd.crs_cross, mgd.crs_matrix, mgd.crs_progeny, mgd.crs_references, mgd.crs_typings, mgd.dag_closure, mgd.dag_dag, mgd.dag_edge, mgd.dag_label, mgd.dag_node, mgd.voc_vocabdag, mgd.go_tracking, mgd.gxd_antibody, mgd.gxd_antigen, mgd.gxd_antibodyalias, mgd.gxd_antibodymarker, mgd.gxd_antibodyprep, mgd.gxd_gellane, mgd.gxd_insituresult, mgd.gxd_insituresultimage, mgd.gxd_assaytype, mgd.gxd_assaynote, mgd.gxd_gelband, mgd.gxd_gelrow, mgd.gxd_genotype, mgd.gxd_gellanestructure, mgd.gxd_theilerstage, mgd.voc_annotheader, mgd.gxd_htexperiment, mgd.gxd_htexperimentvariable, mgd.gxd_htrawsample, mgd.gxd_htsample, mgd.gxd_htsample_rnaseq, mgd.gxd_htsample_rnaseqcombined, mgd.gxd_htsample_rnaseqset, mgd.gxd_htsample_rnaseqset_cache, mgd.gxd_htsample_rnaseqsetmember, mgd.gxd_index_stages, mgd.gxd_isresultcelltype, mgd.img_imagepane, mgd.gxd_isresultstructure, mgd.gxd_probeprep, mgd.prb_probe, mgd.img_imagepane_assoc, mgd.mgi_note, mgd.map_coord_collection, mgd.map_coord_feature, mgd.map_coordinate, mgd.mrk_chromosome, mgd.mgi_dbinfo, mgd.mgi_keyvalue, mgd.mgi_notetype, mgd.mgi_organism_mgitype, mgd.mgi_property, mgd.mgi_propertytype, mgd.mgi_relationship_category, mgd.mgi_relationship_property, mgd.mgi_set, mgd.mgi_setmember, mgd.mgi_setmember_emapa, mgd.mgi_translation, mgd.mgi_translationtype, mgd.voc_vocab, mgd.mld_assay_types, mgd.mld_concordance, mgd.mld_contig, mgd.mld_contigprobe, mgd.mld_expt_marker, mgd.mld_expt_notes, mgd.mld_fish, mgd.mld_fish_region, mgd.mld_hit, mgd.mld_hybrid, mgd.mld_insitu, mgd.mld_isregion, mgd.mld_matrix, mgd.mld_mc2point, mgd.mld_mcdatalist, mgd.mld_ri, mgd.mld_ri2point, mgd.mld_ridata, mgd.mld_statistics, mgd.mrk_types, mgd.mrk_biotypemapping, mgd.mrk_cluster, mgd.mrk_clustermember, mgd.mrk_current, mgd.mrk_history, mgd.mrk_label, mgd.mrk_location_cache, mgd.mrk_status, mgd.mrk_mcv_cache, mgd.mrk_mcv_count_cache, mgd.mrk_notes, mgd.prb_alias, mgd.prb_allele, mgd.prb_allele_strain, mgd.prb_marker, mgd.prb_notes, mgd.prb_ref_notes, mgd.prb_rflv, mgd.prb_tissue, mgd.prb_strain_genotype, mgd.prb_strain_marker, mgd.ri_riset, mgd.ri_summary, mgd.ri_summary_expt_ref, mgd.seq_allele_assoc, mgd.seq_coord_cache, mgd.seq_genemodel, mgd.seq_genetrap, mgd.seq_marker_cache, mgd.seq_probe_cache, mgd.seq_sequence, mgd.seq_sequence_assoc, mgd.seq_sequence_raw, mgd.seq_source_assoc, mgd.voc_allele_cache, mgd.voc_annot_count_cache, mgd.voc_evidence_property, mgd.voc_marker_cache, mgd.voc_term_emapa, mgd.voc_term_emaps, mgd.wks_rosetta" }, { diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index c569bd9c0f..1050fff604 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -388,6 +388,7 @@ func checkSql(sqlInfoArr []sqlInfo, fpath string) { reportCase(fpath, "RANGE with offset PRECEDING/FOLLOWING is not supported for column type numeric and offset type double precision", "https://github.com/yugabyte/yugabyte-db/issues/10692", "", "TABLE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true } else if stmt := fetchRegex.FindStringSubmatch(sqlInfo.stmt); stmt != nil { location := strings.ToUpper(stmt[1]) if slices.Contains(notSupportedFetchLocation, location) { @@ -396,24 +397,31 @@ func checkSql(sqlInfoArr []sqlInfo, fpath string) { "Please verify the DDL on your YugabyteDB version before proceeding", "CURSOR", sqlInfo.objName, sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } } else if stmt := alterAggRegex.FindStringSubmatch(sqlInfo.stmt); stmt != nil { + summaryMap["AGGREGATE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER AGGREGATE not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/2717", "", "AGGREGATE", stmt[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if dropCollRegex.MatchString(sqlInfo.stmt) { + summaryMap["COLLATION"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP COLLATION", sqlInfo.formattedStmt), "COLLATION", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if dropIdxRegex.MatchString(sqlInfo.stmt) { + summaryMap["INDEX"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP INDEX", sqlInfo.formattedStmt), "INDEX", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if dropViewRegex.MatchString(sqlInfo.stmt) { + summaryMap["VIEW"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP VIEW", sqlInfo.formattedStmt), "VIEW", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if dropSeqRegex.MatchString(sqlInfo.stmt) { + summaryMap["SEQUENCE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP SEQUENCE", sqlInfo.formattedStmt), "SEQUENCE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if dropForeignRegex.MatchString(sqlInfo.stmt) { + summaryMap["FOREIGN TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP FOREIGN TABLE", sqlInfo.formattedStmt), "FOREIGN TABLE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if idx := dropIdxConcurRegex.FindStringSubmatch(sqlInfo.stmt); idx != nil { + summaryMap["INDEX"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP INDEX CONCURRENTLY not supported yet", "https://github.com/yugabyte/yugabyte-db/issues/22717", "", "INDEX", idx[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if currentOfRegex.MatchString(sqlInfo.stmt) { @@ -449,63 +457,83 @@ func checkDDL(sqlInfoArr []sqlInfo, fpath string, objType string) { reportCase(fpath, "OIDs are not supported for user tables.", "https://github.com/yugabyte/yugabyte-db/issues/10273", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterOfRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE OF not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterSchemaRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE SET SCHEMA not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/3947", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if createSchemaRegex.MatchString(sqlInfo.stmt) { + summaryMap["SCHEMA"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "CREATE SCHEMA with elements not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/10865", "", "SCHEMA", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterNotOfRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE NOT OF not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterColumnStatsRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE ALTER column SET STATISTICS not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterColumnStorageRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE ALTER column SET STORAGE not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterColumnResetAttributesRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE ALTER column RESET (attribute) not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterConstrRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE ALTER CONSTRAINT not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := setOidsRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE SET WITH OIDS not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[4], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := withoutClusterRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE SET WITHOUT CLUSTER not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterSetRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE SET not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterIdxRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER INDEX SET not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "INDEX", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterResetRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE RESET not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterOptionsRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if typ := dropAttrRegex.FindStringSubmatch(sqlInfo.stmt); typ != nil { + summaryMap["TYPE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TYPE DROP ATTRIBUTE not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1893", "", "TYPE", typ[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if typ := alterTypeRegex.FindStringSubmatch(sqlInfo.stmt); typ != nil { + summaryMap["TYPE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TYPE not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1893", "", "TYPE", typ[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := alterInhRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE INHERIT not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := valConstrRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE VALIDATE CONSTRAINT not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if spc := alterTblSpcRegex.FindStringSubmatch(sqlInfo.stmt); spc != nil { + summaryMap["TABLESPACE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLESPACE not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1153", "", "TABLESPACE", spc[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if spc := alterViewRegex.FindStringSubmatch(sqlInfo.stmt); spc != nil { + summaryMap["VIEW"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER VIEW not supported yet.", "https://github.com/YugaByte/yugabyte-db/issues/1131", "", "VIEW", spc[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := cLangRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { @@ -544,9 +572,11 @@ func checkForeign(sqlInfoArr []sqlInfo, fpath string) { for _, sqlInfo := range sqlInfoArr { //TODO: refactor it later to remove all the unneccessary regexes if tbl := primRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "Primary key constraints are not supported on foreign tables.", "https://github.com/yugabyte/yugabyte-db/issues/10698", "", "TABLE", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } else if tbl := foreignKeyRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { + summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "Foreign key constraints are not supported on foreign tables.", "https://github.com/yugabyte/yugabyte-db/issues/10699", "", "TABLE", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") } @@ -619,15 +649,44 @@ func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fil issueType = UNSUPPORTED_PLPGSQL_OBEJCTS } - //TODO: how to different between same issue on differnt obejct types like ALTER/INDEX for not adding it ot invalid count map - increaseInvalidCount, ok := issueInstance.Details["INCREASE_INVALID_COUNT"] - if !ok || (increaseInvalidCount.(bool)) { - summaryMap[issueInstance.ObjectType].invalidCount[issueInstance.ObjectName] = true + var constraintIssues = []string{ + queryissue.EXCLUSION_CONSTRAINTS, + queryissue.DEFERRABLE_CONSTRAINTS, + queryissue.PK_UK_ON_COMPLEX_DATATYPE, + } + /* + TODO: + // unsupportedIndexIssue + // ObjectType = INDEX + // ObjectName = idx_name ON table_name + // invalidCount.Type = INDEX + // invalidCount.Name = ObjectName (because this is fully qualified) + // DisplayName = ObjectName + + // deferrableConstraintIssue + // ObjectType = TABLE + // ObjectName = table_name + // invalidCount.Type = TABLE + // invalidCount.Name = ObjectName + // DisplayName = table_name (constraint_name) (!= ObjectName) + + // Solutions + // 1. Define a issue.ObjectDisplayName + // 2. Keep it in issue.Details and write logic in UI layer to construct display name. + */ + displayObjectName := issueInstance.ObjectName + + constraintName, ok := issueInstance.Details[queryissue.CONSTRAINT_NAME] + if slices.Contains(constraintIssues, issueInstance.Type) && ok { + //In case of constraint issues we add constraint name to the object name as well + displayObjectName = fmt.Sprintf("%s, constraint: (%s)", issueInstance.ObjectName, constraintName) } + summaryMap[issueInstance.ObjectType].invalidCount[issueInstance.ObjectName] = true + return utils.Issue{ ObjectType: issueInstance.ObjectType, - ObjectName: issueInstance.ObjectName, + ObjectName: displayObjectName, Reason: issueInstance.TypeName, SqlStatement: issueInstance.SqlStatement, DocsLink: issueInstance.DocsLink, diff --git a/yb-voyager/cmd/templates/migration_assessment_report.template b/yb-voyager/cmd/templates/migration_assessment_report.template index 67d95e33c1..215f055efd 100644 --- a/yb-voyager/cmd/templates/migration_assessment_report.template +++ b/yb-voyager/cmd/templates/migration_assessment_report.template @@ -101,7 +101,7 @@ - + {{range .SchemaSummary.DBObjects}} diff --git a/yb-voyager/cmd/templates/schema_analysis_report.html b/yb-voyager/cmd/templates/schema_analysis_report.html index 34a4b6e230..562ee4f50d 100644 --- a/yb-voyager/cmd/templates/schema_analysis_report.html +++ b/yb-voyager/cmd/templates/schema_analysis_report.html @@ -80,7 +80,7 @@

Migration Information

Schema Summary

Object TypeTotal CountTotal Objects Object Names
- + {{ range .SchemaSummary.DBObjects }} {{ if .TotalCount }} diff --git a/yb-voyager/cmd/templates/schema_analysis_report.txt b/yb-voyager/cmd/templates/schema_analysis_report.txt index 2c1bcfffa8..2d046227bf 100644 --- a/yb-voyager/cmd/templates/schema_analysis_report.txt +++ b/yb-voyager/cmd/templates/schema_analysis_report.txt @@ -19,12 +19,12 @@ Migration Complexity : {{ .MigrationComplexity }} Schema Summary --------------- {{ range .SchemaSummary.DBObjects }} -Object Type : {{ .ObjectType }} - - Total Count : {{ .TotalCount }} - - Valid Count : {{ sub .TotalCount .InvalidCount }} - - Invalid Count : {{ .InvalidCount }} - - Object Names : {{ .ObjectNames }}{{ if .Details }} - - Details : {{ .Details }} +Object Type : {{ .ObjectType }} + - Total Objects : {{ .TotalCount }} + - Objects Without Issues : {{ sub .TotalCount .InvalidCount }} + - Objects With Issues : {{ .InvalidCount }} + - Object Names : {{ .ObjectNames }}{{ if .Details }} + - Details : {{ .Details }} {{ end }} {{ end }} diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index 49fdf5423d..dbab60c240 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -56,6 +56,7 @@ const ( // Object types const ( + CONSTRAINT_NAME = "ConstraintName" TABLE_OBJECT_TYPE = "TABLE" FOREIGN_TABLE_OBJECT_TYPE = "FOREIGN TABLE" FUNCTION_OBJECT_TYPE = "FUNCTION" diff --git a/yb-voyager/src/query/queryissue/detectors_ddl.go b/yb-voyager/src/query/queryissue/detectors_ddl.go index 33714d00c6..a791e96066 100644 --- a/yb-voyager/src/query/queryissue/detectors_ddl.go +++ b/yb-voyager/src/query/queryissue/detectors_ddl.go @@ -80,16 +80,18 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss if c.ConstraintType == queryparser.EXCLUSION_CONSTR_TYPE { issues = append(issues, NewExclusionConstraintIssue( obj.GetObjectType(), - fmt.Sprintf("%s, constraint: (%s)", table.GetObjectName(), c.ConstraintName), + table.GetObjectName(), "", + c.ConstraintName, )) } if c.ConstraintType != queryparser.FOREIGN_CONSTR_TYPE && c.IsDeferrable { issues = append(issues, NewDeferrableConstraintIssue( obj.GetObjectType(), - fmt.Sprintf("%s, constraint: (%s)", table.GetObjectName(), c.ConstraintName), + table.GetObjectName(), "", + c.ConstraintName, )) } @@ -106,10 +108,10 @@ func (d *TableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIss } issues = append(issues, NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue( obj.GetObjectType(), - fmt.Sprintf("%s, constraint: (%s)", table.GetObjectName(), c.ConstraintName), + table.GetObjectName(), "", typeName, - true, + c.ConstraintName, )) } } @@ -413,15 +415,17 @@ func (aid *AlterTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Q if alter.ConstraintType == queryparser.EXCLUSION_CONSTR_TYPE { issues = append(issues, NewExclusionConstraintIssue( obj.GetObjectType(), - fmt.Sprintf("%s, constraint: (%s)", alter.GetObjectName(), alter.ConstraintName), + alter.GetObjectName(), "", + alter.ConstraintName, )) } if alter.ConstraintType != queryparser.FOREIGN_CONSTR_TYPE && alter.IsDeferrable { issues = append(issues, NewDeferrableConstraintIssue( obj.GetObjectType(), - fmt.Sprintf("%s, constraint: (%s)", alter.GetObjectName(), alter.ConstraintName), + alter.GetObjectName(), "", + alter.ConstraintName, )) } @@ -447,10 +451,10 @@ func (aid *AlterTableIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Q } issues = append(issues, NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue( obj.GetObjectType(), - fmt.Sprintf("%s, constraint: (%s)", alter.GetObjectName(), alter.ConstraintName), + alter.GetObjectName(), "", typeName, - false, + alter.ConstraintName, )) } diff --git a/yb-voyager/src/query/queryissue/issues_ddl.go b/yb-voyager/src/query/queryissue/issues_ddl.go index 08fe2c69c1..b131eb62b1 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl.go +++ b/yb-voyager/src/query/queryissue/issues_ddl.go @@ -51,10 +51,6 @@ var unloggedTableIssue = issue.Issue{ func NewUnloggedTableIssue(objectType string, objectName string, sqlStatement string) QueryIssue { details := map[string]interface{}{} - //for UNLOGGED TABLE as its not reported in the TABLE objects - if objectType == "TABLE" { - details["INCREASE_INVALID_COUNT"] = false - } return newQueryIssue(unloggedTableIssue, objectType, objectName, sqlStatement, details) } @@ -81,10 +77,6 @@ var storageParameterIssue = issue.Issue{ func NewStorageParameterIssue(objectType string, objectName string, sqlStatement string) QueryIssue { details := map[string]interface{}{} - //for ALTER AND INDEX both same struct now how to differentiate which one to not - if objectType == "TABLE" { - details["INCREASE_INVALID_COUNT"] = false - } return newQueryIssue(storageParameterIssue, objectType, objectName, sqlStatement, details) } @@ -98,10 +90,6 @@ var setAttributeIssue = issue.Issue{ func NewSetAttributeIssue(objectType string, objectName string, sqlStatement string) QueryIssue { details := map[string]interface{}{} - //for ALTER AND INDEX both same struct now how to differentiate which one to not - if objectType == "TABLE" { - details["INCREASE_INVALID_COUNT"] = false - } return newQueryIssue(setAttributeIssue, objectType, objectName, sqlStatement, details) } @@ -115,10 +103,6 @@ var clusterOnIssue = issue.Issue{ func NewClusterONIssue(objectType string, objectName string, sqlStatement string) QueryIssue { details := map[string]interface{}{} - //for ALTER AND INDEX both same struct now how to differentiate which one to not - if objectType == "TABLE" { - details["INCREASE_INVALID_COUNT"] = false - } return newQueryIssue(clusterOnIssue, objectType, objectName, sqlStatement, details) } @@ -132,10 +116,6 @@ var disableRuleIssue = issue.Issue{ func NewDisableRuleIssue(objectType string, objectName string, sqlStatement string, ruleName string) QueryIssue { details := map[string]interface{}{} - //for ALTER AND INDEX both same struct now how to differentiate which one to not - if objectType == "TABLE" { - details["INCREASE_INVALID_COUNT"] = false - } issue := disableRuleIssue issue.Suggestion = fmt.Sprintf(issue.Suggestion, ruleName) return newQueryIssue(issue, objectType, objectName, sqlStatement, details) @@ -149,8 +129,11 @@ var exclusionConstraintIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#exclusion-constraints-is-not-supported", } -func NewExclusionConstraintIssue(objectType string, objectName string, sqlStatement string) QueryIssue { - return newQueryIssue(exclusionConstraintIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +func NewExclusionConstraintIssue(objectType string, objectName string, sqlStatement string, constraintName string) QueryIssue { + details := map[string]interface{}{ + CONSTRAINT_NAME: constraintName, + } + return newQueryIssue(exclusionConstraintIssue, objectType, objectName, sqlStatement, details) } var deferrableConstraintIssue = issue.Issue{ @@ -161,8 +144,11 @@ var deferrableConstraintIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#deferrable-constraint-on-constraints-other-than-foreign-keys-is-not-supported", } -func NewDeferrableConstraintIssue(objectType string, objectName string, sqlStatement string) QueryIssue { - return newQueryIssue(deferrableConstraintIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +func NewDeferrableConstraintIssue(objectType string, objectName string, sqlStatement string, constraintName string) QueryIssue { + details := map[string]interface{}{ + CONSTRAINT_NAME: constraintName, + } + return newQueryIssue(deferrableConstraintIssue, objectType, objectName, sqlStatement, details) } var multiColumnGinIndexIssue = issue.Issue{ @@ -195,10 +181,10 @@ var policyRoleIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", } -func NewPolicyRoleIssue(objectType string, objectName string, SqlStatement string, roles []string) QueryIssue { +func NewPolicyRoleIssue(objectType string, objectName string, sqlStatement string, roles []string) QueryIssue { issue := policyRoleIssue issue.TypeName = fmt.Sprintf("%s Users - (%s)", issue.TypeName, strings.Join(roles, ",")) - return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var constraintTriggerIssue = issue.Issue{ @@ -208,13 +194,9 @@ var constraintTriggerIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#constraint-trigger-is-not-supported", } -func NewConstraintTriggerIssue(objectType string, objectName string, SqlStatement string) QueryIssue { +func NewConstraintTriggerIssue(objectType string, objectName string, sqlStatement string) QueryIssue { details := map[string]interface{}{} - //for CONSTRAINT TRIGGER we don't have separate object type TODO: fix - if objectType == "TRIGGER" { - details["INCREASE_INVALID_COUNT"] = false - } - return newQueryIssue(constraintTriggerIssue, objectType, objectName, SqlStatement, details) + return newQueryIssue(constraintTriggerIssue, objectType, objectName, sqlStatement, details) } var referencingClauseInTriggerIssue = issue.Issue{ @@ -224,8 +206,8 @@ var referencingClauseInTriggerIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#referencing-clause-for-triggers", } -func NewReferencingClauseTrigIssue(objectType string, objectName string, SqlStatement string) QueryIssue { - return newQueryIssue(referencingClauseInTriggerIssue, objectType, objectName, SqlStatement, map[string]interface{}{}) +func NewReferencingClauseTrigIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(referencingClauseInTriggerIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var beforeRowTriggerOnPartitionTableIssue = issue.Issue{ @@ -236,8 +218,8 @@ var beforeRowTriggerOnPartitionTableIssue = issue.Issue{ Suggestion: "Create the triggers on individual partitions.", } -func NewBeforeRowOnPartitionTableIssue(objectType string, objectName string, SqlStatement string) QueryIssue { - return newQueryIssue(beforeRowTriggerOnPartitionTableIssue, objectType, objectName, SqlStatement, map[string]interface{}{}) +func NewBeforeRowOnPartitionTableIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(beforeRowTriggerOnPartitionTableIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var alterTableAddPKOnPartitionIssue = issue.Issue{ @@ -252,13 +234,9 @@ var alterTableAddPKOnPartitionIssue = issue.Issue{ }, } -func NewAlterTableAddPKOnPartiionIssue(objectType string, objectName string, SqlStatement string) QueryIssue { +func NewAlterTableAddPKOnPartiionIssue(objectType string, objectName string, sqlStatement string) QueryIssue { details := map[string]interface{}{} - //for ALTER AND INDEX both same struct now how to differentiate which one to not - if objectType == "TABLE" { - details["INCREASE_INVALID_COUNT"] = false - } - return newQueryIssue(alterTableAddPKOnPartitionIssue, objectType, objectName, SqlStatement, details) + return newQueryIssue(alterTableAddPKOnPartitionIssue, objectType, objectName, sqlStatement, details) } var expressionPartitionIssue = issue.Issue{ @@ -269,8 +247,8 @@ var expressionPartitionIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/mysql/#tables-partitioned-with-expressions-cannot-contain-primary-unique-keys", } -func NewExpressionPartitionIssue(objectType string, objectName string, SqlStatement string) QueryIssue { - return newQueryIssue(expressionPartitionIssue, objectType, objectName, SqlStatement, map[string]interface{}{}) +func NewExpressionPartitionIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(expressionPartitionIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var multiColumnListPartition = issue.Issue{ @@ -281,8 +259,8 @@ var multiColumnListPartition = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/mysql/#multi-column-partition-by-list-is-not-supported", } -func NewMultiColumnListPartition(objectType string, objectName string, SqlStatement string) QueryIssue { - return newQueryIssue(multiColumnListPartition, objectType, objectName, SqlStatement, map[string]interface{}{}) +func NewMultiColumnListPartition(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(multiColumnListPartition, objectType, objectName, sqlStatement, map[string]interface{}{}) } var insufficientColumnsInPKForPartition = issue.Issue{ @@ -293,10 +271,10 @@ var insufficientColumnsInPKForPartition = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/oracle/#partition-key-column-not-part-of-primary-key-columns", } -func NewInsufficientColumnInPKForPartition(objectType string, objectName string, SqlStatement string, partitionColumnsNotInPK []string) QueryIssue { +func NewInsufficientColumnInPKForPartition(objectType string, objectName string, sqlStatement string, partitionColumnsNotInPK []string) QueryIssue { issue := insufficientColumnsInPKForPartition issue.TypeName = fmt.Sprintf("%s - (%s)", issue.TypeName, strings.Join(partitionColumnsNotInPK, ", ")) - return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var xmlDatatypeIssue = issue.Issue{ @@ -307,10 +285,10 @@ var xmlDatatypeIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported", } -func NewXMLDatatypeIssue(objectType string, objectName string, SqlStatement string, colName string) QueryIssue { +func NewXMLDatatypeIssue(objectType string, objectName string, sqlStatement string, colName string) QueryIssue { issue := xmlDatatypeIssue issue.TypeName = fmt.Sprintf("%s on column - %s", issue.TypeName, colName) - return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var xidDatatypeIssue = issue.Issue{ @@ -321,10 +299,10 @@ var xidDatatypeIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xid-functions-is-not-supported", } -func NewXIDDatatypeIssue(objectType string, objectName string, SqlStatement string, colName string) QueryIssue { +func NewXIDDatatypeIssue(objectType string, objectName string, sqlStatement string, colName string) QueryIssue { issue := xidDatatypeIssue issue.TypeName = fmt.Sprintf("%s on column - %s", issue.TypeName, colName) - return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var postgisDatatypeIssue = issue.Issue{ @@ -334,10 +312,10 @@ var postgisDatatypeIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb", } -func NewPostGisDatatypeIssue(objectType string, objectName string, SqlStatement string, typeName string, colName string) QueryIssue { +func NewPostGisDatatypeIssue(objectType string, objectName string, sqlStatement string, typeName string, colName string) QueryIssue { issue := postgisDatatypeIssue issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) - return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var unsupportedDatatypesIssue = issue.Issue{ @@ -347,10 +325,10 @@ var unsupportedDatatypesIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb", } -func NewUnsupportedDatatypesIssue(objectType string, objectName string, SqlStatement string, typeName string, colName string) QueryIssue { +func NewUnsupportedDatatypesIssue(objectType string, objectName string, sqlStatement string, typeName string, colName string) QueryIssue { issue := unsupportedDatatypesIssue issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) - return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var unsupportedDatatypesForLiveMigrationIssue = issue.Issue{ @@ -360,10 +338,10 @@ var unsupportedDatatypesForLiveMigrationIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", } -func NewUnsupportedDatatypesForLMIssue(objectType string, objectName string, SqlStatement string, typeName string, colName string) QueryIssue { +func NewUnsupportedDatatypesForLMIssue(objectType string, objectName string, sqlStatement string, typeName string, colName string) QueryIssue { issue := unsupportedDatatypesForLiveMigrationIssue issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) - return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var unsupportedDatatypesForLiveMigrationWithFFOrFBIssue = issue.Issue{ @@ -373,10 +351,10 @@ var unsupportedDatatypesForLiveMigrationWithFFOrFBIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", } -func NewUnsupportedDatatypesForLMWithFFOrFBIssue(objectType string, objectName string, SqlStatement string, typeName string, colName string) QueryIssue { +func NewUnsupportedDatatypesForLMWithFFOrFBIssue(objectType string, objectName string, sqlStatement string, typeName string, colName string) QueryIssue { issue := unsupportedDatatypesForLiveMigrationWithFFOrFBIssue issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) - return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var primaryOrUniqueOnUnsupportedIndexTypesIssue = issue.Issue{ @@ -387,15 +365,13 @@ var primaryOrUniqueOnUnsupportedIndexTypesIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", //Keeping it similar for now, will see if we need to a separate issue on docs, } -func NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue(objectType string, objectName string, SqlStatement string, typeName string, increaseInvalidCnt bool) QueryIssue { - details := map[string]interface{}{} - //for ALTER not increasing count, but for Create increasing TODO: fix - if !increaseInvalidCnt { - details["INCREASE_INVALID_COUNT"] = false +func NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue(objectType string, objectName string, sqlStatement string, typeName string, constraintName string) QueryIssue { + details := map[string]interface{}{ + CONSTRAINT_NAME: constraintName, } issue := primaryOrUniqueOnUnsupportedIndexTypesIssue issue.TypeName = fmt.Sprintf(issue.TypeName, typeName) - return newQueryIssue(issue, objectType, objectName, SqlStatement, details) + return newQueryIssue(issue, objectType, objectName, sqlStatement, details) } var indexOnComplexDatatypesIssue = issue.Issue{ @@ -406,10 +382,10 @@ var indexOnComplexDatatypesIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", } -func NewIndexOnComplexDatatypesIssue(objectType string, objectName string, SqlStatement string, typeName string) QueryIssue { +func NewIndexOnComplexDatatypesIssue(objectType string, objectName string, sqlStatement string, typeName string) QueryIssue { issue := indexOnComplexDatatypesIssue issue.TypeName = fmt.Sprintf(issue.TypeName, typeName) - return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var foreignTableIssue = issue.Issue{ @@ -420,10 +396,10 @@ var foreignTableIssue = issue.Issue{ DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping", } -func NewForeignTableIssue(objectType string, objectName string, SqlStatement string, serverName string) QueryIssue { +func NewForeignTableIssue(objectType string, objectName string, sqlStatement string, serverName string) QueryIssue { issue := foreignTableIssue issue.Suggestion = fmt.Sprintf(issue.Suggestion, serverName) - return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var inheritanceIssue = issue.Issue{ diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index 99d989b528..cf2a95eedb 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -191,7 +191,7 @@ func TestAllIssues(t *testing.T) { NewDisableRuleIssue("TABLE", "public.example", stmt4, "example_rule"), }, stmt5: []QueryIssue{ - NewDeferrableConstraintIssue("TABLE", "abc, constraint: (cnstr_id)", stmt5), + NewDeferrableConstraintIssue("TABLE", "abc", stmt5, "cnstr_id"), }, stmt6: []QueryIssue{ NewAdvisoryLocksIssue("DML_QUERY", "", stmt6), @@ -210,8 +210,8 @@ func TestAllIssues(t *testing.T) { NewInsufficientColumnInPKForPartition("TABLE", "test_non_pk_multi_column_list", stmt10, []string{"country_code", "record_type"}), }, stmt11: []QueryIssue{ - NewExclusionConstraintIssue("TABLE", "Test, constraint: (Test_room_id_time_range_excl)", stmt11), - NewExclusionConstraintIssue("TABLE", "Test, constraint: (no_time_overlap_constr)", stmt11), + NewExclusionConstraintIssue("TABLE", "Test", stmt11, "Test_room_id_time_range_excl"), + NewExclusionConstraintIssue("TABLE", "Test", stmt11, "no_time_overlap_constr"), }, stmt13: []QueryIssue{ NewIndexOnComplexDatatypesIssue("INDEX", "idx_on_daterange ON test_dt", stmt13, "daterange"), @@ -259,7 +259,7 @@ func TestDDLIssues(t *testing.T) { }, stmt16: []QueryIssue{ NewXmlFunctionsIssue("TABLE", "public.xml_data_example", stmt16), - NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue("TABLE", "public.xml_data_example, constraint: (xml_data_example_d_key)", stmt16, "daterange", true), + NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue("TABLE", "public.xml_data_example", stmt16, "daterange", "xml_data_example_d_key"), NewMultiColumnListPartition("TABLE", "public.xml_data_example", stmt16), NewInsufficientColumnInPKForPartition("TABLE", "public.xml_data_example", stmt16, []string{"name"}), NewXMLDatatypeIssue("TABLE", "public.xml_data_example", stmt16, "description"), diff --git a/yb-voyager/src/utils/commonVariables.go b/yb-voyager/src/utils/commonVariables.go index c2b0abb8c9..4c18b78fad 100644 --- a/yb-voyager/src/utils/commonVariables.go +++ b/yb-voyager/src/utils/commonVariables.go @@ -90,6 +90,7 @@ type SchemaSummary struct { DBObjects []DBObject `json:"DatabaseObjects"` } +//TODO: Rename the variables of TotalCount and InvalidCount -> TotalObjects and ObjectsWithIssues type DBObject struct { ObjectType string `json:"ObjectType"` TotalCount int `json:"TotalCount"`
ObjectTotal CountValid CountInvalid CountObject NamesDetails
ObjectTotal ObjectsObjects Without IssuesObjects With IssuesObject NamesDetails