From b9489425685cf16b1729c4b305ea51b44b80e691 Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Mon, 23 Dec 2024 11:12:44 +0530 Subject: [PATCH 01/17] Updated code to use pg_query_go v6 instead of v5 (#2103) --- .../expected_files/expectedAssessmentReport.json | 7 ++++++- .../expected_schema_analysis_report.json | 14 +++++++++++++- yb-voyager/cmd/analyzeSchema.go | 2 +- yb-voyager/cmd/exportSchema.go | 2 +- yb-voyager/go.mod | 4 ++-- yb-voyager/go.sum | 4 ++-- yb-voyager/src/query/queryparser/ddl_processor.go | 2 +- .../src/query/queryparser/helpers_protomsg.go | 2 +- yb-voyager/src/query/queryparser/helpers_struct.go | 2 +- yb-voyager/src/query/queryparser/query_parser.go | 2 +- 10 files changed, 29 insertions(+), 12 deletions(-) diff --git a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json index 8aa82381e..082f0d501 100644 --- a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json @@ -36,7 +36,7 @@ { "ObjectType": "FUNCTION", "TotalCount": 139, - "InvalidCount": 16, + "InvalidCount": 17, "ObjectNames": "mgd.acc_accession_delete, mgd.acc_assignj, mgd.acc_assignmgi, mgd.acc_delete_byacckey, mgd.acc_insert, mgd.acc_setmax, mgd.acc_split, mgd.acc_update, mgd.accref_insert, mgd.accref_process, mgd.all_allele_delete, mgd.all_allele_insert, mgd.all_allele_update, mgd.all_cellline_delete, mgd.all_cellline_update1, mgd.all_cellline_update2, mgd.all_convertallele, mgd.all_createwildtype, mgd.all_insertallele, mgd.all_mergeallele, mgd.all_mergewildtypes, mgd.all_reloadlabel, mgd.all_variant_delete, mgd.bib_keepwfrelevance, mgd.bib_refs_delete, mgd.bib_refs_insert, mgd.bib_reloadcache, mgd.bib_updatewfstatusap, mgd.bib_updatewfstatusgo, mgd.bib_updatewfstatusgxd, mgd.bib_updatewfstatusqtl, mgd.gxd_addcelltypeset, mgd.gxd_addemapaset, mgd.gxd_addgenotypeset, mgd.gxd_allelepair_insert, mgd.gxd_antibody_delete, mgd.gxd_antibody_insert, mgd.gxd_antigen_delete, mgd.gxd_antigen_insert, mgd.gxd_assay_delete, mgd.gxd_assay_insert, mgd.gxd_checkduplicategenotype, mgd.gxd_gelrow_insert, mgd.gxd_genotype_delete, mgd.gxd_genotype_insert, mgd.gxd_getgenotypesdatasets, mgd.gxd_getgenotypesdatasetscount, mgd.gxd_htexperiment_delete, mgd.gxd_htrawsample_delete, mgd.gxd_htsample_ageminmax, mgd.gxd_index_insert, mgd.gxd_index_insert_before, mgd.gxd_index_update, mgd.gxd_orderallelepairs, mgd.gxd_ordergenotypes, mgd.gxd_ordergenotypesall, mgd.gxd_ordergenotypesmissing, mgd.gxd_removebadgelband, mgd.gxd_replacegenotype, mgd.img_image_delete, mgd.img_image_insert, mgd.img_setpdo, mgd.mgi_addsetmember, mgd.mgi_checkemapaclipboard, mgd.mgi_cleannote, mgd.mgi_deleteemapaclipboarddups, mgd.mgi_insertreferenceassoc, mgd.mgi_insertsynonym, mgd.mgi_mergerelationship, mgd.mgi_organism_delete, mgd.mgi_organism_insert, mgd.mgi_processnote, mgd.mgi_reference_assoc_delete, mgd.mgi_reference_assoc_insert, mgd.mgi_relationship_delete, mgd.mgi_resetageminmax, mgd.mgi_resetsequencenum, mgd.mgi_setmember_emapa_insert, mgd.mgi_setmember_emapa_update, mgd.mgi_statistic_delete, mgd.mgi_updatereferenceassoc, mgd.mgi_updatesetmember, mgd.mld_expt_marker_update, mgd.mld_expts_delete, mgd.mld_expts_insert, mgd.mrk_allelewithdrawal, mgd.mrk_cluster_delete, mgd.mrk_copyhistory, mgd.mrk_deletewithdrawal, mgd.mrk_inserthistory, mgd.mrk_marker_delete, mgd.mrk_marker_insert, mgd.mrk_marker_update, mgd.mrk_mergewithdrawal, mgd.mrk_reloadlocation, mgd.mrk_reloadreference, mgd.mrk_simplewithdrawal, mgd.mrk_strainmarker_delete, mgd.mrk_updatekeys, mgd.prb_ageminmax, mgd.prb_getstrainbyreference, mgd.prb_getstraindatasets, mgd.prb_getstrainreferences, mgd.prb_insertreference, mgd.prb_marker_insert, mgd.prb_marker_update, mgd.prb_mergestrain, mgd.prb_probe_delete, mgd.prb_probe_insert, mgd.prb_processanonymoussource, mgd.prb_processprobesource, mgd.prb_processseqloadersource, mgd.prb_processsequencesource, mgd.prb_reference_delete, mgd.prb_reference_update, mgd.prb_setstrainreview, mgd.prb_source_delete, mgd.prb_strain_delete, mgd.prb_strain_insert, mgd.seq_deletebycreatedby, mgd.seq_deletedummy, mgd.seq_deleteobsoletedummy, mgd.seq_merge, mgd.seq_sequence_delete, mgd.seq_split, mgd.voc_annot_insert, mgd.voc_copyannotevidencenotes, mgd.voc_evidence_delete, mgd.voc_evidence_insert, mgd.voc_evidence_property_delete, mgd.voc_evidence_update, mgd.voc_mergeannotations, mgd.voc_mergedupannotations, mgd.voc_mergeterms, mgd.voc_processannotheader, mgd.voc_processannotheaderall, mgd.voc_processannotheadermissing, mgd.voc_resetterms, mgd.voc_term_delete" }, { @@ -13780,6 +13780,11 @@ "ObjectType": "FUNCTION", "ObjectName": "mgd.seq_split", "SqlStatement": "acc_accession.accID%TYPE" + }, + { + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mgi_resetageminmax", + "SqlStatement": "prb_source.age%TYPE" } ], "MinimumVersionsFixedIn": null, diff --git a/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json index cfe0307c1..bc5c2eee9 100644 --- a/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/mgi/expected_files/expected_schema_analysis_report.json @@ -36,7 +36,7 @@ { "ObjectType": "FUNCTION", "TotalCount": 139, - "InvalidCount": 16, + "InvalidCount": 17, "ObjectNames": "mgd.acc_accession_delete, mgd.acc_assignj, mgd.acc_assignmgi, mgd.acc_delete_byacckey, mgd.acc_insert, mgd.acc_setmax, mgd.acc_split, mgd.acc_update, mgd.accref_insert, mgd.accref_process, mgd.all_allele_delete, mgd.all_allele_insert, mgd.all_allele_update, mgd.all_cellline_delete, mgd.all_cellline_update1, mgd.all_cellline_update2, mgd.all_convertallele, mgd.all_createwildtype, mgd.all_insertallele, mgd.all_mergeallele, mgd.all_mergewildtypes, mgd.all_reloadlabel, mgd.all_variant_delete, mgd.bib_keepwfrelevance, mgd.bib_refs_delete, mgd.bib_refs_insert, mgd.bib_reloadcache, mgd.bib_updatewfstatusap, mgd.bib_updatewfstatusgo, mgd.bib_updatewfstatusgxd, mgd.bib_updatewfstatusqtl, mgd.gxd_addcelltypeset, mgd.gxd_addemapaset, mgd.gxd_addgenotypeset, mgd.gxd_allelepair_insert, mgd.gxd_antibody_delete, mgd.gxd_antibody_insert, mgd.gxd_antigen_delete, mgd.gxd_antigen_insert, mgd.gxd_assay_delete, mgd.gxd_assay_insert, mgd.gxd_checkduplicategenotype, mgd.gxd_gelrow_insert, mgd.gxd_genotype_delete, mgd.gxd_genotype_insert, mgd.gxd_getgenotypesdatasets, mgd.gxd_getgenotypesdatasetscount, mgd.gxd_htexperiment_delete, mgd.gxd_htrawsample_delete, mgd.gxd_htsample_ageminmax, mgd.gxd_index_insert, mgd.gxd_index_insert_before, mgd.gxd_index_update, mgd.gxd_orderallelepairs, mgd.gxd_ordergenotypes, mgd.gxd_ordergenotypesall, mgd.gxd_ordergenotypesmissing, mgd.gxd_removebadgelband, mgd.gxd_replacegenotype, mgd.img_image_delete, mgd.img_image_insert, mgd.img_setpdo, mgd.mgi_addsetmember, mgd.mgi_checkemapaclipboard, mgd.mgi_cleannote, mgd.mgi_deleteemapaclipboarddups, mgd.mgi_insertreferenceassoc, mgd.mgi_insertsynonym, mgd.mgi_mergerelationship, mgd.mgi_organism_delete, mgd.mgi_organism_insert, mgd.mgi_processnote, mgd.mgi_reference_assoc_delete, mgd.mgi_reference_assoc_insert, mgd.mgi_relationship_delete, mgd.mgi_resetageminmax, mgd.mgi_resetsequencenum, mgd.mgi_setmember_emapa_insert, mgd.mgi_setmember_emapa_update, mgd.mgi_statistic_delete, mgd.mgi_updatereferenceassoc, mgd.mgi_updatesetmember, mgd.mld_expt_marker_update, mgd.mld_expts_delete, mgd.mld_expts_insert, mgd.mrk_allelewithdrawal, mgd.mrk_cluster_delete, mgd.mrk_copyhistory, mgd.mrk_deletewithdrawal, mgd.mrk_inserthistory, mgd.mrk_marker_delete, mgd.mrk_marker_insert, mgd.mrk_marker_update, mgd.mrk_mergewithdrawal, mgd.mrk_reloadlocation, mgd.mrk_reloadreference, mgd.mrk_simplewithdrawal, mgd.mrk_strainmarker_delete, mgd.mrk_updatekeys, mgd.prb_ageminmax, mgd.prb_getstrainbyreference, mgd.prb_getstraindatasets, mgd.prb_getstrainreferences, mgd.prb_insertreference, mgd.prb_marker_insert, mgd.prb_marker_update, mgd.prb_mergestrain, mgd.prb_probe_delete, mgd.prb_probe_insert, mgd.prb_processanonymoussource, mgd.prb_processprobesource, mgd.prb_processseqloadersource, mgd.prb_processsequencesource, mgd.prb_reference_delete, mgd.prb_reference_update, mgd.prb_setstrainreview, mgd.prb_source_delete, mgd.prb_strain_delete, mgd.prb_strain_insert, mgd.seq_deletebycreatedby, mgd.seq_deletedummy, mgd.seq_deleteobsoletedummy, mgd.seq_merge, mgd.seq_sequence_delete, mgd.seq_split, mgd.voc_annot_insert, mgd.voc_copyannotevidencenotes, mgd.voc_evidence_delete, mgd.voc_evidence_insert, mgd.voc_evidence_property_delete, mgd.voc_evidence_update, mgd.voc_mergeannotations, mgd.voc_mergedupannotations, mgd.voc_mergeterms, mgd.voc_processannotheader, mgd.voc_processannotheaderall, mgd.voc_processannotheadermissing, mgd.voc_resetterms, mgd.voc_term_delete" }, { @@ -1325,6 +1325,18 @@ "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_plpgsql_objects", + "ObjectType": "FUNCTION", + "ObjectName": "mgd.mgi_resetageminmax", + "Reason": "Referenced type declaration of variables", + "SqlStatement": "prb_source.age%TYPE", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/mgi/export-dir/schema/functions/function.sql", + "Suggestion": "Fix the syntax to include the actual type name instead of referencing the type of a column", + "GH": "https://github.com/yugabyte/yugabyte-db/issues/23619", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + "MinimumVersionsFixedIn": null } ] } diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 0c8d21cc6..80a4d1462 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -27,7 +27,7 @@ import ( "strings" "text/template" - pg_query "github.com/pganalyze/pg_query_go/v5" + pg_query "github.com/pganalyze/pg_query_go/v6" "github.com/samber/lo" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" diff --git a/yb-voyager/cmd/exportSchema.go b/yb-voyager/cmd/exportSchema.go index 4adcdca1a..092a87c39 100644 --- a/yb-voyager/cmd/exportSchema.go +++ b/yb-voyager/cmd/exportSchema.go @@ -22,7 +22,7 @@ import ( "strings" "github.com/fatih/color" - pg_query "github.com/pganalyze/pg_query_go/v5" + pg_query "github.com/pganalyze/pg_query_go/v6" "github.com/samber/lo" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" diff --git a/yb-voyager/go.mod b/yb-voyager/go.mod index c833375e2..a8a306dbc 100644 --- a/yb-voyager/go.mod +++ b/yb-voyager/go.mod @@ -11,6 +11,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.18.15 github.com/aws/aws-sdk-go-v2/service/s3 v1.30.5 github.com/davecgh/go-spew v1.1.1 + github.com/deckarep/golang-set/v2 v2.7.0 github.com/docker/go-connections v0.5.0 github.com/dustin/go-humanize v1.0.1 github.com/fatih/color v1.13.0 @@ -29,7 +30,7 @@ require ( github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2 github.com/mitchellh/go-ps v1.0.0 github.com/nightlyone/lockfile v1.0.0 - github.com/pganalyze/pg_query_go/v5 v5.1.0 + github.com/pganalyze/pg_query_go/v6 v6.0.0 github.com/samber/lo v1.38.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.6.1 @@ -55,7 +56,6 @@ require ( github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect - github.com/deckarep/golang-set/v2 v2.7.0 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/docker v27.1.1+incompatible // indirect github.com/docker/go-units v0.5.0 // indirect diff --git a/yb-voyager/go.sum b/yb-voyager/go.sum index 44ee2af8d..3b156d1c0 100644 --- a/yb-voyager/go.sum +++ b/yb-voyager/go.sum @@ -1711,8 +1711,8 @@ github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwb github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= github.com/performancecopilot/speed/v4 v4.0.0/go.mod h1:qxrSyuDGrTOWfV+uKRFhfxw6h/4HXRGUiZiufxo49BM= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pganalyze/pg_query_go/v5 v5.1.0 h1:MlxQqHZnvA3cbRQYyIrjxEjzo560P6MyTgtlaf3pmXg= -github.com/pganalyze/pg_query_go/v5 v5.1.0/go.mod h1:FsglvxidZsVN+Ltw3Ai6nTgPVcK2BPukH3jCDEqc1Ug= +github.com/pganalyze/pg_query_go/v6 v6.0.0 h1:in6RkR/apfqlAtvqgDxd4Y4o87a5Pr8fkKDB4DrDo2c= +github.com/pganalyze/pg_query_go/v6 v6.0.0/go.mod h1:nvTHIuoud6e1SfrUaFwHqT0i4b5Nr+1rPWVds3B5+50= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= diff --git a/yb-voyager/src/query/queryparser/ddl_processor.go b/yb-voyager/src/query/queryparser/ddl_processor.go index 22f60fd48..2d2a6e766 100644 --- a/yb-voyager/src/query/queryparser/ddl_processor.go +++ b/yb-voyager/src/query/queryparser/ddl_processor.go @@ -20,7 +20,7 @@ import ( "slices" "strings" - pg_query "github.com/pganalyze/pg_query_go/v5" + pg_query "github.com/pganalyze/pg_query_go/v6" "github.com/samber/lo" ) diff --git a/yb-voyager/src/query/queryparser/helpers_protomsg.go b/yb-voyager/src/query/queryparser/helpers_protomsg.go index 88b3a85d6..0eb609d15 100644 --- a/yb-voyager/src/query/queryparser/helpers_protomsg.go +++ b/yb-voyager/src/query/queryparser/helpers_protomsg.go @@ -18,7 +18,7 @@ package queryparser import ( "strings" - pg_query "github.com/pganalyze/pg_query_go/v5" + pg_query "github.com/pganalyze/pg_query_go/v6" log "github.com/sirupsen/logrus" "google.golang.org/protobuf/reflect/protoreflect" ) diff --git a/yb-voyager/src/query/queryparser/helpers_struct.go b/yb-voyager/src/query/queryparser/helpers_struct.go index 67b4294af..eafc90791 100644 --- a/yb-voyager/src/query/queryparser/helpers_struct.go +++ b/yb-voyager/src/query/queryparser/helpers_struct.go @@ -19,7 +19,7 @@ import ( "fmt" "strings" - pg_query "github.com/pganalyze/pg_query_go/v5" + pg_query "github.com/pganalyze/pg_query_go/v6" "github.com/samber/lo" ) diff --git a/yb-voyager/src/query/queryparser/query_parser.go b/yb-voyager/src/query/queryparser/query_parser.go index 724643683..864cb30d0 100644 --- a/yb-voyager/src/query/queryparser/query_parser.go +++ b/yb-voyager/src/query/queryparser/query_parser.go @@ -29,7 +29,7 @@ package queryparser import ( "fmt" - pg_query "github.com/pganalyze/pg_query_go/v5" + pg_query "github.com/pganalyze/pg_query_go/v6" log "github.com/sirupsen/logrus" ) From 47fe19820f384a5dad17a99318a676c4ec55e399 Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Mon, 23 Dec 2024 15:59:24 +0530 Subject: [PATCH 02/17] Add regex functions issue (#2100) - Detect and report unsupported regex functions --- .github/workflows/misc-migtests.yml | 3 +- .../expectedAssessmentReport.json | 5 ++ .../expectedAssessmentReport.json | 5 ++ .../expectedAssessmentReport.json | 14 ++++- .../pg_assessment_report.sql | 5 +- .../expectedAssessmentReport.json | 5 ++ .../expectedAssessmentReport.json | 5 ++ .../expectedAssessmentReport.json | 5 ++ .../expectedAssessmentReport.json | 5 ++ .../expectedAssessmentReport.json | 6 +- .../expectedAssessmentReport.json | 5 ++ .../expectedAssessmentReport.json | 5 ++ .../expectedAssessmentReport.json | 5 ++ yb-voyager/cmd/analyzeSchema.go | 1 + yb-voyager/cmd/assessMigrationCommand.go | 63 ++++++++++--------- yb-voyager/cmd/constants.go | 1 + yb-voyager/src/query/queryissue/constants.go | 2 + yb-voyager/src/query/queryissue/detectors.go | 10 ++- yb-voyager/src/query/queryissue/helpers.go | 4 ++ yb-voyager/src/query/queryissue/issues_dml.go | 13 ++++ .../src/query/queryissue/issues_dml_test.go | 22 +++++++ .../queryissue/parser_issue_detector_test.go | 30 +++++++++ yb-voyager/src/utils/commonVariables.go | 3 +- 23 files changed, 184 insertions(+), 38 deletions(-) diff --git a/.github/workflows/misc-migtests.yml b/.github/workflows/misc-migtests.yml index add010cdd..cd9fe7ad3 100644 --- a/.github/workflows/misc-migtests.yml +++ b/.github/workflows/misc-migtests.yml @@ -12,7 +12,7 @@ jobs: services: postgres: - image: postgres:14 + image: postgres:15 env: POSTGRES_PASSWORD: secret # Set health checks to wait until postgres has started @@ -59,6 +59,7 @@ jobs: run: | cd installer_scripts yes | ./install-yb-voyager --install-from-local-source --only-pg-support + echo "/usr/lib/postgresql/16/bin" >> "$GITHUB_PATH" env: ON_INSTALLER_ERROR_OUTPUT_LOG: Y diff --git a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json index 85faaee1c..509e19e38 100755 --- a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json @@ -741,6 +741,11 @@ "FeatureName": "Large Object Functions", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Regex Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json index a76703008..0c9360ba3 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json @@ -176,6 +176,11 @@ "FeatureName": "System Columns", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Regex Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 931e2611e..c7e3cb30c 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -45,7 +45,7 @@ { "ObjectType": "TABLE", "TotalCount": 66, - "InvalidCount": 22, + "InvalidCount": 23, "ObjectNames": "public.ordersentry, public.library_nested, public.orders_lateral, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.sales_region, public.boston, public.c, public.parent_table, public.child_table, public.citext_type, public.combined_tbl, public.documents, public.employees2, public.ext_test, public.foo, public.inet_type, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.orders, public.orders2, public.products, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.sales_region, schema2.boston, schema2.c, schema2.parent_table, schema2.child_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.orders, schema2.orders2, schema2.products, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" }, { @@ -590,6 +590,16 @@ } ], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Regex Functions", + "Objects": [ + { + "ObjectName": "public.ordersentry", + "SqlStatement": "CREATE TABLE public.ordersentry (\n order_id integer NOT NULL,\n customer_name text NOT NULL,\n product_name text NOT NULL,\n quantity integer NOT NULL,\n price numeric(10,2) NOT NULL,\n processed_at timestamp without time zone,\n r integer DEFAULT regexp_count('This is an example. Another example. Example is a common word.'::text, 'example'::text)\n);" + } + ], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", @@ -1158,7 +1168,7 @@ "SchemaName": "public", "ObjectName": "ordersentry", "RowCount": 6, - "ColumnCount": 6, + "ColumnCount": 7, "Reads": 0, "Writes": 6, "ReadsPerSecond": 0, diff --git a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql index b08d4f1ac..0c089922b 100644 --- a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql +++ b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql @@ -242,7 +242,8 @@ EXECUTE FUNCTION public.check_sales_region(); product_name TEXT NOT NULL, quantity INT NOT NULL, price NUMERIC(10, 2) NOT NULL, - processed_at timestamp + processed_at timestamp, + r INT DEFAULT regexp_count('This is an example. Another example. Example is a common word.', 'example') -- regex functions in default ); INSERT INTO public.ordersentry (customer_name, product_name, quantity, price) @@ -360,4 +361,4 @@ BEGIN PERFORM lo_unlink(loid); END IF; END; -$$ LANGUAGE plpgsql; \ No newline at end of file +$$ LANGUAGE plpgsql; diff --git a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json index 082f0d501..329c9404b 100644 --- a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json @@ -622,6 +622,11 @@ "FeatureName": "Unlogged tables", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Regex Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json index 5b804b142..a553ba272 100755 --- a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json @@ -683,6 +683,11 @@ "FeatureName": "Unlogged tables", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Regex Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json index ebc254029..b03097dac 100755 --- a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json @@ -209,6 +209,11 @@ "FeatureName": "Unlogged tables", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Regex Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json index 65c1989f2..dcea98ff7 100755 --- a/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json @@ -204,6 +204,11 @@ "FeatureName": "View with check option", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Regex Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json index 945e5e203..3178c50ab 100644 --- a/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json @@ -943,8 +943,12 @@ "FeatureName": "Large Object Functions", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Regex Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } - ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ diff --git a/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json index d2a1b9165..ed5bc11f9 100755 --- a/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json @@ -274,6 +274,11 @@ "FeatureName": "Unlogged tables", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Regex Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json index 141da784d..0c32b640e 100755 --- a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json @@ -206,6 +206,11 @@ "FeatureName": "View with check option", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Regex Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json index 412bcdf95..4bef6f17a 100644 --- a/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json @@ -384,6 +384,11 @@ "FeatureName": "Large Object Functions", "Objects": [], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Regex Functions", + "Objects": [], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 80a4d1462..85d1ce25c 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -688,6 +688,7 @@ func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fil ObjectType: issueInstance.ObjectType, ObjectName: displayObjectName, Reason: issueInstance.TypeName, + Type: issueInstance.Type, SqlStatement: issueInstance.SqlStatement, DocsLink: issueInstance.DocsLink, FilePath: fileName, diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 305ec5653..21e006c08 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -947,7 +947,7 @@ func areMinVersionsFixedInEqual(m1 map[string]*ybversion.YBVersion, m2 map[strin return true } -func getUnsupportedFeaturesFromSchemaAnalysisReport(featureName string, issueReason string, schemaAnalysisReport utils.SchemaReport, displayDDLInHTML bool, description string) UnsupportedFeature { +func getUnsupportedFeaturesFromSchemaAnalysisReport(featureName string, issueReason string, issueType string, schemaAnalysisReport utils.SchemaReport, displayDDLInHTML bool, description string) UnsupportedFeature { log.Info("filtering issues for feature: ", featureName) objects := make([]ObjectInfo, 0) link := "" // for oracle we shouldn't display any line for links @@ -958,7 +958,9 @@ func getUnsupportedFeaturesFromSchemaAnalysisReport(featureName string, issueRea if !slices.Contains([]string{UNSUPPORTED_FEATURES, MIGRATION_CAVEATS}, issue.IssueType) { continue } - if strings.Contains(issue.Reason, issueReason) { + issueMatched := lo.Ternary[bool](issueType != "", issueType == issue.Type, strings.Contains(issue.Reason, issueReason)) + + if issueMatched { objectInfo := ObjectInfo{ ObjectName: issue.ObjectName, SqlStatement: issue.SqlStatement, @@ -984,33 +986,34 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem displayIndexMethod := strings.ToUpper(indexMethod) feature := fmt.Sprintf("%s indexes", displayIndexMethod) reason := fmt.Sprintf(INDEX_METHOD_ISSUE_REASON, displayIndexMethod) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(feature, reason, schemaAnalysisReport, false, "")) - } - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CONSTRAINT_TRIGGERS_FEATURE, CONSTRAINT_TRIGGER_ISSUE_REASON, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(INHERITED_TABLES_FEATURE, INHERITANCE_ISSUE_REASON, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(GENERATED_COLUMNS_FEATURE, STORED_GENERATED_COLUMN_ISSUE_REASON, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CONVERSIONS_OBJECTS_FEATURE, CONVERSION_ISSUE_REASON, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(MULTI_COLUMN_GIN_INDEX_FEATURE, GIN_INDEX_MULTI_COLUMN_ISSUE_REASON, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(ALTER_SETTING_ATTRIBUTE_FEATURE, ALTER_TABLE_SET_ATTRIBUTE_ISSUE, schemaAnalysisReport, true, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(DISABLING_TABLE_RULE_FEATURE, ALTER_TABLE_DISABLE_RULE_ISSUE, schemaAnalysisReport, true, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CLUSTER_ON_FEATURE, ALTER_TABLE_CLUSTER_ON_ISSUE, schemaAnalysisReport, true, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(STORAGE_PARAMETERS_FEATURE, STORAGE_PARAMETERS_DDL_STMT_ISSUE, schemaAnalysisReport, true, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(EXTENSION_FEATURE, UNSUPPORTED_EXTENSION_ISSUE, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(EXCLUSION_CONSTRAINT_FEATURE, EXCLUSION_CONSTRAINT_ISSUE, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(DEFERRABLE_CONSTRAINT_FEATURE, DEFERRABLE_CONSTRAINT_ISSUE, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(VIEW_CHECK_FEATURE, VIEW_CHECK_OPTION_ISSUE, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(feature, reason, "", schemaAnalysisReport, false, "")) + } + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CONSTRAINT_TRIGGERS_FEATURE, CONSTRAINT_TRIGGER_ISSUE_REASON, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(INHERITED_TABLES_FEATURE, INHERITANCE_ISSUE_REASON, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(GENERATED_COLUMNS_FEATURE, STORED_GENERATED_COLUMN_ISSUE_REASON, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CONVERSIONS_OBJECTS_FEATURE, CONVERSION_ISSUE_REASON, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(MULTI_COLUMN_GIN_INDEX_FEATURE, GIN_INDEX_MULTI_COLUMN_ISSUE_REASON, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(ALTER_SETTING_ATTRIBUTE_FEATURE, ALTER_TABLE_SET_ATTRIBUTE_ISSUE, "", schemaAnalysisReport, true, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(DISABLING_TABLE_RULE_FEATURE, ALTER_TABLE_DISABLE_RULE_ISSUE, "", schemaAnalysisReport, true, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CLUSTER_ON_FEATURE, ALTER_TABLE_CLUSTER_ON_ISSUE, "", schemaAnalysisReport, true, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(STORAGE_PARAMETERS_FEATURE, STORAGE_PARAMETERS_DDL_STMT_ISSUE, "", schemaAnalysisReport, true, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(EXTENSION_FEATURE, UNSUPPORTED_EXTENSION_ISSUE, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(EXCLUSION_CONSTRAINT_FEATURE, EXCLUSION_CONSTRAINT_ISSUE, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(DEFERRABLE_CONSTRAINT_FEATURE, DEFERRABLE_CONSTRAINT_ISSUE, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(VIEW_CHECK_FEATURE, VIEW_CHECK_OPTION_ISSUE, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisReport, queryissue.UnsupportedIndexDatatypes)) pkOrUkConstraintIssuePrefix := strings.Split(ISSUE_PK_UK_CONSTRAINT_WITH_COMPLEX_DATATYPES, "%s")[0] - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(PK_UK_CONSTRAINT_ON_COMPLEX_DATATYPES_FEATURE, pkOrUkConstraintIssuePrefix, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(PK_UK_CONSTRAINT_ON_COMPLEX_DATATYPES_FEATURE, pkOrUkConstraintIssuePrefix, "", schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(UNLOGGED_TABLE_FEATURE, ISSUE_UNLOGGED_TABLE, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REFERENCING_TRIGGER_FEATURE, REFERENCING_CLAUSE_FOR_TRIGGERS, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE_FEATURE, BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.ADVISORY_LOCKS_NAME, queryissue.ADVISORY_LOCKS_NAME, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.XML_FUNCTIONS_NAME, queryissue.XML_FUNCTIONS_NAME, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SYSTEM_COLUMNS_NAME, queryissue.SYSTEM_COLUMNS_NAME, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.LARGE_OBJECT_FUNCTIONS_NAME, queryissue.LARGE_OBJECT_FUNCTIONS_NAME, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(UNLOGGED_TABLE_FEATURE, ISSUE_UNLOGGED_TABLE, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REFERENCING_TRIGGER_FEATURE, REFERENCING_CLAUSE_FOR_TRIGGERS, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE_FEATURE, BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.ADVISORY_LOCKS_NAME, queryissue.ADVISORY_LOCKS_NAME, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.XML_FUNCTIONS_NAME, queryissue.XML_FUNCTIONS_NAME, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SYSTEM_COLUMNS_NAME, queryissue.SYSTEM_COLUMNS_NAME, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.LARGE_OBJECT_FUNCTIONS_NAME, queryissue.LARGE_OBJECT_FUNCTIONS_NAME, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REGEX_FUNCTIONS_FEATURE, "", queryissue.REGEX_FUNCTIONS, schemaAnalysisReport, false, "")) return unsupportedFeatures, nil } @@ -1025,7 +1028,7 @@ func getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisiReport utils.Schem unsupportedIndexDatatypes = append(unsupportedIndexDatatypes, "array") // adding it here only as we know issue form analyze will come with type unsupportedIndexDatatypes = append(unsupportedIndexDatatypes, "user_defined_type") // adding it here as we UDTs will come with this type. for _, unsupportedType := range unsupportedIndexDatatypes { - indexes := getUnsupportedFeaturesFromSchemaAnalysisReport(fmt.Sprintf("%s indexes", unsupportedType), fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, unsupportedType), schemaAnalysisReport, false, "") + indexes := getUnsupportedFeaturesFromSchemaAnalysisReport(fmt.Sprintf("%s indexes", unsupportedType), fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, unsupportedType), "", schemaAnalysisReport, false, "") for _, object := range indexes.Objects { formattedObject := object formattedObject.ObjectName = fmt.Sprintf("%s: %s", strings.ToUpper(unsupportedType), object.ObjectName) @@ -1042,7 +1045,7 @@ func getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisiReport utils.Schem func fetchUnsupportedOracleFeaturesFromSchemaReport(schemaAnalysisReport utils.SchemaReport) ([]UnsupportedFeature, error) { log.Infof("fetching unsupported features for Oracle...") unsupportedFeatures := make([]UnsupportedFeature, 0) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(COMPOUND_TRIGGER_FEATURE, COMPOUND_TRIGGER_ISSUE_REASON, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(COMPOUND_TRIGGER_FEATURE, COMPOUND_TRIGGER_ISSUE_REASON, "", schemaAnalysisReport, false, "")) return unsupportedFeatures, nil } @@ -1382,11 +1385,11 @@ func addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration case POSTGRESQL: log.Infof("add migration caveats to assessment report") migrationCaveats := make([]UnsupportedFeature, 0) - migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(ALTER_PARTITION_ADD_PK_CAVEAT_FEATURE, ADDING_PK_TO_PARTITIONED_TABLE_ISSUE_REASON, + migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(ALTER_PARTITION_ADD_PK_CAVEAT_FEATURE, ADDING_PK_TO_PARTITIONED_TABLE_ISSUE_REASON, "", schemaAnalysisReport, true, DESCRIPTION_ADD_PK_TO_PARTITION_TABLE)) - migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(FOREIGN_TABLE_CAVEAT_FEATURE, FOREIGN_TABLE_ISSUE_REASON, + migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(FOREIGN_TABLE_CAVEAT_FEATURE, FOREIGN_TABLE_ISSUE_REASON, "", schemaAnalysisReport, false, DESCRIPTION_FOREIGN_TABLES)) - migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(POLICIES_CAVEAT_FEATURE, POLICY_ROLE_ISSUE, + migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(POLICIES_CAVEAT_FEATURE, POLICY_ROLE_ISSUE, "", schemaAnalysisReport, false, DESCRIPTION_POLICY_ROLE_ISSUE)) if len(unsupportedDataTypesForLiveMigration) > 0 { diff --git a/yb-voyager/cmd/constants.go b/yb-voyager/cmd/constants.go index 85c9f225e..b22891ae0 100644 --- a/yb-voyager/cmd/constants.go +++ b/yb-voyager/cmd/constants.go @@ -213,6 +213,7 @@ const ( REFERENCING_TRIGGER_FEATURE = "REFERENCING clause for triggers" BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE_FEATURE = "BEFORE ROW triggers on Partitioned tables" PK_UK_CONSTRAINT_ON_COMPLEX_DATATYPES_FEATURE = "Primary / Unique key constraints on complex datatypes" + REGEX_FUNCTIONS_FEATURE = "Regex Functions" // Migration caveats diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index e07367a38..3a6d42453 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -59,6 +59,8 @@ const ( ADVISORY_LOCKS_NAME = "Advisory Locks" SYSTEM_COLUMNS_NAME = "System Columns" XML_FUNCTIONS_NAME = "XML Functions" + + REGEX_FUNCTIONS = "REGEX_FUNCTIONS" ) // Object types diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index 228496a69..bb9490786 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -32,9 +32,10 @@ type UnsupportedConstructDetector interface { type FuncCallDetector struct { query string - // right now it covers Advisory Locks and XML functions + advisoryLocksFuncsDetected mapset.Set[string] xmlFuncsDetected mapset.Set[string] + regexFuncsDetected mapset.Set[string] loFuncsDetected mapset.Set[string] } @@ -43,6 +44,7 @@ func NewFuncCallDetector(query string) *FuncCallDetector { query: query, advisoryLocksFuncsDetected: mapset.NewThreadUnsafeSet[string](), xmlFuncsDetected: mapset.NewThreadUnsafeSet[string](), + regexFuncsDetected: mapset.NewThreadUnsafeSet[string](), loFuncsDetected: mapset.NewThreadUnsafeSet[string](), } } @@ -62,6 +64,9 @@ func (d *FuncCallDetector) Detect(msg protoreflect.Message) error { if unsupportedXmlFunctions.ContainsOne(funcName) { d.xmlFuncsDetected.Add(funcName) } + if unsupportedRegexFunctions.ContainsOne(funcName) { + d.regexFuncsDetected.Add(funcName) + } if unsupportedLargeObjectFunctions.ContainsOne(funcName) { d.loFuncsDetected.Add(funcName) @@ -78,6 +83,9 @@ func (d *FuncCallDetector) GetIssues() []QueryIssue { if d.xmlFuncsDetected.Cardinality() > 0 { issues = append(issues, NewXmlFunctionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) } + if d.regexFuncsDetected.Cardinality() > 0 { + issues = append(issues, NewRegexFunctionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) + } if d.loFuncsDetected.Cardinality() > 0 { issues = append(issues, NewLOFuntionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) } diff --git a/yb-voyager/src/query/queryissue/helpers.go b/yb-voyager/src/query/queryissue/helpers.go index 01bacd4b7..617b2fb70 100644 --- a/yb-voyager/src/query/queryissue/helpers.go +++ b/yb-voyager/src/query/queryissue/helpers.go @@ -58,6 +58,10 @@ var unsupportedXmlFunctions = mapset.NewThreadUnsafeSet([]string{ "xmlconcat2", "xmlvalidate", "xml_in", "xml_out", "xml_recv", "xml_send", // System XML I/O }...) +var unsupportedRegexFunctions = mapset.NewThreadUnsafeSet([]string{ + "regexp_count", "regexp_instr", "regexp_like", +}...) + var UnsupportedIndexMethods = []string{ "gist", "brin", diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index 3528ead45..33db98be0 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -57,6 +57,19 @@ func NewXmlFunctionsIssue(objectType string, objectName string, sqlStatement str return newQueryIssue(xmlFunctionsIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } +var regexFunctionsIssue = issue.Issue{ + Type: REGEX_FUNCTIONS, + TypeName: "Regex Functions", + TypeDescription: "", + Suggestion: "", + GH: "", + DocsLink: "", +} + +func NewRegexFunctionsIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(regexFunctionsIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} + var loFunctionsIssue = issue.Issue{ Type: LARGE_OBJECT_FUNCTIONS, TypeName: LARGE_OBJECT_FUNCTIONS_NAME, diff --git a/yb-voyager/src/query/queryissue/issues_dml_test.go b/yb-voyager/src/query/queryissue/issues_dml_test.go index 8f7c6610d..af3d8afcc 100644 --- a/yb-voyager/src/query/queryissue/issues_dml_test.go +++ b/yb-voyager/src/query/queryissue/issues_dml_test.go @@ -40,6 +40,25 @@ func testLOFunctionsIssue(t *testing.T) { assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "Transaction for catalog table write operation 'pg_largeobject_metadata' not found", loDatatypeIssue) } +func testRegexFunctionsIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + + stmts := []string{ + `SELECT regexp_count('This is an example. Another example. Example is a common word.', 'example')`, + `SELECT regexp_instr('This is an example. Another example. Example is a common word.', 'example')`, + `SELECT regexp_like('This is an example. Another example. Example is a common word.', 'example')`, + } + + for _, stmt := range stmts { + _, err = conn.Exec(ctx, stmt) + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "does not exist", regexFunctionsIssue) + } +} + func TestDMLIssuesInYBVersion(t *testing.T) { var err error ybVersion := os.Getenv("YB_VERSION") @@ -68,4 +87,7 @@ func TestDMLIssuesInYBVersion(t *testing.T) { success := t.Run(fmt.Sprintf("%s-%s", "lo functions", ybVersion), testLOFunctionsIssue) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "regex functions", ybVersion), testRegexFunctionsIssue) + assert.True(t, success) + } diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index 9ba4dac4d..e7c979118 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -485,3 +485,33 @@ func TestSingleXMLIssueIsDetected(t *testing.T) { fatalIfError(t, err) assert.Equal(t, 1, len(issues)) } + +func TestRegexFunctionsIssue(t *testing.T) { + dmlStmts := []string{ + `SELECT regexp_count('This is an example. Another example. Example is a common word.', 'example')`, + `SELECT regexp_instr('This is an example. Another example. Example is a common word.', 'example')`, + `SELECT regexp_like('This is an example. Another example. Example is a common word.', 'example')`, + `SELECT regexp_count('abc','abc'), regexp_instr('abc','abc'), regexp_like('abc','abc')`, + } + + ddlStmts := []string{ + `CREATE TABLE x (id INT PRIMARY KEY, id2 INT DEFAULT regexp_count('This is an example. Another example. Example is a common word.', 'example'))`, + } + + parserIssueDetector := NewParserIssueDetector() + + for _, stmt := range dmlStmts { + issues, err := parserIssueDetector.getDMLIssues(stmt) + fatalIfError(t, err) + assert.Equal(t, 1, len(issues)) + assert.Equal(t, NewRegexFunctionsIssue(DML_QUERY_OBJECT_TYPE, "", stmt), issues[0]) + } + + for _, stmt := range ddlStmts { + issues, err := parserIssueDetector.getDDLIssues(stmt) + fatalIfError(t, err) + assert.Equal(t, 1, len(issues)) + assert.Equal(t, NewRegexFunctionsIssue(TABLE_OBJECT_TYPE, "x", stmt), issues[0]) + } + +} diff --git a/yb-voyager/src/utils/commonVariables.go b/yb-voyager/src/utils/commonVariables.go index 4c18b78fa..c1d51326b 100644 --- a/yb-voyager/src/utils/commonVariables.go +++ b/yb-voyager/src/utils/commonVariables.go @@ -101,10 +101,11 @@ type DBObject struct { // TODO: support MinimumVersionsFixedIn in xml type Issue struct { - IssueType string `json:"IssueType"` + IssueType string `json:"IssueType"` //category: unsupported_features, unsupported_plpgsql_objects, etc ObjectType string `json:"ObjectType"` ObjectName string `json:"ObjectName"` Reason string `json:"Reason"` + Type string `json:"-" xml:"-"` // identifier for issue type ADVISORY_LOCKS, SYSTEM_COLUMNS, etc SqlStatement string `json:"SqlStatement,omitempty"` FilePath string `json:"FilePath"` Suggestion string `json:"Suggestion"` From ecb4c05a15e7d68a6b9e6e64f94d10390b590436 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Mon, 23 Dec 2024 17:39:45 +0530 Subject: [PATCH 03/17] Do not add the feature in UnsupportedFeatures field if there are no objects for it in the JSON report (#2110) --- .../expectedChild2AssessmentReport.json | 28 +--- .../expectedAssessmentReport.json | 141 +--------------- .../expectedAssessmentReport.json | 154 +----------------- .../expectedAssessmentReport.json | 57 ------- .../expectedAssessmentReport.json | 148 +---------------- .../expectedAssessmentReport.json | 103 +----------- .../expectedAssessmentReport.json | 143 +--------------- .../expectedAssessmentReport.json | 145 +---------------- .../expectedAssessmentReport.json | 148 +---------------- .../expectedAssessmentReport.json | 140 +--------------- .../expectedAssessmentReport.json | 145 +---------------- .../expectedAssessmentReport.json | 148 +---------------- yb-voyager/cmd/assessMigrationCommand.go | 33 +++- 13 files changed, 40 insertions(+), 1493 deletions(-) diff --git a/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild2AssessmentReport.json b/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild2AssessmentReport.json index 297ad902e..8e99cbde6 100644 --- a/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild2AssessmentReport.json +++ b/migtests/tests/oracle/bulk-assessment-test/expected_reports/expectedChild2AssessmentReport.json @@ -106,33 +106,7 @@ }, "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", - "UnsupportedFeatures": [ - { - "FeatureName": "Compound Triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unsupported Indexes", - "Objects": null, - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Virtual Columns", - "Objects": null, - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Inherited Types", - "Objects": null, - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unsupported Partitioning Methods", - "Objects": null, - "MinimumVersionsFixedIn": null - } - ], + "UnsupportedFeatures": null, "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ { diff --git a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json index 509e19e38..90979b4b3 100755 --- a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json @@ -309,66 +309,6 @@ ], "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ - { - "FeatureName": "GIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BRIN indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "SPGIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Constraint triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "REFERENCING clause for triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Inherited tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Tables with stored generated columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Conversion objects", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Gin indexes on multi-columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Setting attribute=value on column", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Disabling rule on table", - "Objects": [], - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Clustering table on index", "Objects": [ @@ -648,16 +588,6 @@ "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", "MinimumVersionsFixedIn": null }, - { - "FeatureName": "Advisory Locks", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName":"System Columns", - "Objects":[], - "MinimumVersionsFixedIn":null - }, { "FeatureName": "XML Functions", "Objects": [ @@ -696,56 +626,6 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Storage parameters in DDLs", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Extensions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Exclusion constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Deferrable constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "View with check option", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Index on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unlogged tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Large Object Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Regex Functions", - "Objects": [], - "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", @@ -1774,26 +1654,7 @@ } ], "Notes": null, - "MigrationCaveats": [ - { - "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Foreign tables", - "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Policies", - "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", - "MinimumVersionsFixedIn": null - } - ], + "MigrationCaveats": null, "UnsupportedQueryConstructs": null, "UnsupportedPlPgSqlObjects": null } diff --git a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json index 0c9360ba3..bbcf4613d 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json @@ -51,138 +51,7 @@ }, "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", - "UnsupportedFeatures": [ - { - "FeatureName": "GIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BRIN indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "SPGIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Constraint triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Inherited tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Tables with stored generated columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Conversion objects", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Gin indexes on multi-columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Setting attribute=value on column", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Disabling rule on table", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Clustering table on index", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Storage parameters in DDLs", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Extensions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Exclusion constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Large Object Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Deferrable constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "View with check option", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Index on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unlogged tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "REFERENCING clause for triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Advisory Locks", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "XML Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "System Columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Regex Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - } - ], + "UnsupportedFeatures": null, "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ { @@ -215,26 +84,7 @@ } ], "Notes": null, - "MigrationCaveats": [ - { - "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Foreign tables", - "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Policies", - "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", - "MinimumVersionsFixedIn": null - } - ], + "MigrationCaveats": null, "UnsupportedQueryConstructs": [ { "ConstructTypeName": "Advisory Locks", diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index c7e3cb30c..edfe82ecd 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -360,31 +360,6 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#generated-always-as-stored-type-column-is-not-supported", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Conversion objects", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Gin indexes on multi-columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Setting attribute=value on column", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Disabling rule on table", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "REFERENCING clause for triggers", - "Objects": [], - "MinimumVersionsFixedIn": null }, { "FeatureName": "BEFORE ROW triggers on Partitioned tables", @@ -397,21 +372,6 @@ "DocsLink":"https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#before-row-triggers-on-partitioned-tables", "MinimumVersionsFixedIn": null }, - { - "FeatureName": "Clustering table on index", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Storage parameters in DDLs", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Extensions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Exclusion constraints", "Objects": [ @@ -543,11 +503,6 @@ "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", "MinimumVersionsFixedIn": null }, - { - "FeatureName": "Unlogged tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, { "FeatureName": "System Columns", "Objects": [ @@ -2009,18 +1964,6 @@ "There are some Unlogged tables in the schema. They will be created as regular LOGGED tables in YugabyteDB as unlogged tables are not supported." ], "MigrationCaveats": [ - { - "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Foreign tables", - "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Policies", "Objects": [ diff --git a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json index 329c9404b..c5d63e027 100644 --- a/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json @@ -248,76 +248,6 @@ "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ - { - "FeatureName": "Advisory Locks", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName":"System Columns", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName": "XML Functions", - "Objects": [], -"MinimumVersionsFixedIn":null - }, - { - "FeatureName": "GIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BRIN indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "SPGIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Constraint triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Inherited tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Tables with stored generated columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Conversion objects", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Gin indexes on multi-columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Large Object Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Setting attribute=value on column", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Disabling rule on table", - "Objects": [], - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Clustering table on index", "Objects": [ @@ -572,62 +502,7 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Storage parameters in DDLs", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Extensions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Exclusion constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "REFERENCING clause for triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Deferrable constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "View with check option", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Index on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unlogged tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Regex Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - } + } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ @@ -13541,26 +13416,7 @@ } ], "Notes": null, - "MigrationCaveats": [ - { - "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Foreign tables", - "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Policies", - "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", - "MinimumVersionsFixedIn": null - } - ], + "MigrationCaveats": null, "UnsupportedQueryConstructs": null, "UnsupportedPlPgSqlObjects": [ { diff --git a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json index a553ba272..dd49b1dc6 100755 --- a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json @@ -493,21 +493,6 @@ "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ - { - "FeatureName": "Advisory Locks", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName":"System Columns", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName": "XML Functions", - "Objects": [], -"MinimumVersionsFixedIn":null - }, { "FeatureName": "GIST indexes", "Objects": [ @@ -518,26 +503,6 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BRIN indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "SPGIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Large Object Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Constraint triggers", - "Objects": [], - "MinimumVersionsFixedIn": null }, { "FeatureName": "Inherited tables", @@ -561,16 +526,6 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "REFERENCING clause for triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [], - "MinimumVersionsFixedIn": null }, { "FeatureName": "Tables with stored generated columns", @@ -597,26 +552,6 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#create-or-alter-conversion-is-not-supported", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Gin indexes on multi-columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Setting attribute=value on column", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Disabling rule on table", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Clustering table on index", - "Objects": [], - "MinimumVersionsFixedIn": null }, { "FeatureName": "Storage parameters in DDLs", @@ -636,21 +571,6 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Extensions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Exclusion constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Deferrable constraints", - "Objects": [], - "MinimumVersionsFixedIn": null }, { "FeatureName": "View with check option", @@ -662,11 +582,6 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#view-with-check-option-is-not-supported", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null }, { "FeatureName": "Index on complex datatypes", @@ -678,17 +593,7 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unlogged tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Regex Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - } + } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ @@ -5553,12 +5458,6 @@ ], "Notes": null, "MigrationCaveats": [ - { - "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Foreign tables", "Objects": [ diff --git a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json index b03097dac..3aaf994a3 100755 --- a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json @@ -73,21 +73,6 @@ ], "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ - { - "FeatureName": "Advisory Locks", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName":"System Columns", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName": "XML Functions", - "Objects": [], -"MinimumVersionsFixedIn":null - }, { "FeatureName": "GIST indexes", "Objects": [ @@ -99,76 +84,6 @@ "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported", "MinimumVersionsFixedIn": null }, - { - "FeatureName": "BRIN indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "SPGIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "REFERENCING clause for triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Constraint triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Inherited tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Tables with stored generated columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Conversion objects", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Gin indexes on multi-columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Setting attribute=value on column", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Disabling rule on table", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Clustering table on index", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Storage parameters in DDLs", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Large Object Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Extensions", "Objects": [ @@ -179,42 +94,7 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/explore/ysql-language-features/pg-extensions/", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Exclusion constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Deferrable constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "View with check option", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Index on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unlogged tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Regex Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - } + } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ @@ -346,26 +226,7 @@ } ], "Notes": null, - "MigrationCaveats": [ - { - "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Foreign tables", - "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Policies", - "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", - "MinimumVersionsFixedIn": null - } - ], + "MigrationCaveats": null, "UnsupportedQueryConstructs": null, "UnsupportedPlPgSqlObjects": null } diff --git a/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json index dcea98ff7..3ebfc9a50 100755 --- a/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json @@ -79,138 +79,7 @@ }, "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", - "UnsupportedFeatures": [ - { - "FeatureName": "Advisory Locks", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName":"System Columns", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName": "XML Functions", - "Objects": [], -"MinimumVersionsFixedIn":null - }, - { - "FeatureName": "GIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BRIN indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "SPGIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Constraint triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Inherited tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "REFERENCING clause for triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Tables with stored generated columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Conversion objects", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Large Object Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Gin indexes on multi-columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Index on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unlogged tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Setting attribute=value on column", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Disabling rule on table", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Clustering table on index", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Storage parameters in DDLs", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Extensions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Exclusion constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Deferrable constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "View with check option", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Regex Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - } - ], + "UnsupportedFeatures": null, "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ { @@ -244,12 +113,6 @@ ], "Notes": null, "MigrationCaveats": [ - { - "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Foreign tables", "Objects": [ @@ -265,12 +128,6 @@ "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping", "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Policies", - "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", - "MinimumVersionsFixedIn": null }, { "FeatureName": "Unsupported Data Types for Live Migration with Fall-forward/Fallback", diff --git a/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json index 3178c50ab..4294cd126 100644 --- a/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json @@ -377,41 +377,6 @@ "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ - { - "FeatureName": "Advisory Locks", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName":"System Columns", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName": "XML Functions", - "Objects": [], -"MinimumVersionsFixedIn":null - }, - { - "FeatureName": "GIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BRIN indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "SPGIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Constraint triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Inherited tables", "Objects": [ @@ -858,97 +823,7 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Tables with stored generated columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Conversion objects", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Gin indexes on multi-columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "REFERENCING clause for triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Setting attribute=value on column", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Disabling rule on table", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Clustering table on index", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Storage parameters in DDLs", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Extensions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Exclusion constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Deferrable constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "View with check option", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Index on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unlogged tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Large Object Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Regex Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - } + } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ @@ -26630,26 +26505,7 @@ } ], "Notes": null, - "MigrationCaveats": [ - { - "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Foreign tables", - "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Policies", - "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", - "MinimumVersionsFixedIn": null - } - ], + "MigrationCaveats": null, "UnsupportedQueryConstructs": null, "UnsupportedPlPgSqlObjects": null } diff --git a/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json index ed5bc11f9..8ff126cd0 100755 --- a/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json @@ -118,21 +118,6 @@ "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ - { - "FeatureName": "Advisory Locks", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName":"System Columns", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName": "XML Functions", - "Objects": [], -"MinimumVersionsFixedIn":null - }, { "FeatureName": "GIST indexes", "Objects": [ @@ -143,31 +128,6 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BRIN indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "SPGIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Constraint triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "REFERENCING clause for triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [], - "MinimumVersionsFixedIn": null }, { "FeatureName": "Inherited tables", @@ -199,87 +159,7 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Large Object Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Tables with stored generated columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Conversion objects", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Gin indexes on multi-columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Setting attribute=value on column", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Disabling rule on table", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Clustering table on index", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Storage parameters in DDLs", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Extensions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Exclusion constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Deferrable constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "View with check option", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Index on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unlogged tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Regex Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - } + } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ @@ -986,24 +866,6 @@ ], "Notes": null, "MigrationCaveats": [ - { - "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Foreign tables", - "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Policies", - "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Unsupported Data Types for Live Migration with Fall-forward/Fallback", "Objects": [ diff --git a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json index 0c32b640e..47797ad8d 100755 --- a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json @@ -76,116 +76,6 @@ "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ - { - "FeatureName": "Advisory Locks", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName":"System Columns", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName": "XML Functions", - "Objects": [], -"MinimumVersionsFixedIn":null - }, - { - "FeatureName": "GIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BRIN indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "SPGIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Constraint triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Inherited tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Tables with stored generated columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Large Object Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Conversion objects", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Gin indexes on multi-columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Setting attribute=value on column", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Disabling rule on table", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Clustering table on index", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Storage parameters in DDLs", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Extensions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Index on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unlogged tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "REFERENCING clause for triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Exclusion constraints", "Objects": [ @@ -196,22 +86,7 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#exclusion-constraints-is-not-supported", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Deferrable constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "View with check option", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Regex Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - } + } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ @@ -386,24 +261,6 @@ ], "Notes": null, "MigrationCaveats": [ - { - "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Foreign tables", - "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Policies", - "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Unsupported Data Types for Live Migration with Fall-forward/Fallback", "Objects": [ diff --git a/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json index 4bef6f17a..34c9ea429 100644 --- a/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json @@ -82,86 +82,6 @@ "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", "UnsupportedFeatures": [ - { - "FeatureName": "Advisory Locks", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName":"System Columns", - "Objects":[], - "MinimumVersionsFixedIn":null - }, - { - "FeatureName": "XML Functions", - "Objects": [], -"MinimumVersionsFixedIn":null - }, - { - "FeatureName": "GIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BRIN indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "SPGIST indexes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Constraint triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Inherited tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "REFERENCING clause for triggers", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "BEFORE ROW triggers on Partitioned tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Tables with stored generated columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Conversion objects", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Gin indexes on multi-columns", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Setting attribute=value on column", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Disabling rule on table", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Clustering table on index", - "Objects": [], - "MinimumVersionsFixedIn": null - }, { "FeatureName": "Storage parameters in DDLs", "Objects": [ @@ -344,52 +264,7 @@ ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Extensions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Exclusion constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Deferrable constraints", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "View with check option", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Primary / Unique key constraints on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Index on complex datatypes", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Unlogged tables", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Large Object Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Regex Functions", - "Objects": [], - "MinimumVersionsFixedIn": null - } + } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ @@ -1305,26 +1180,7 @@ } ], "Notes": null, - "MigrationCaveats": [ - { - "FeatureName": "Alter partitioned tables to add Primary Key", - "Objects": [], - "FeatureDescription": "After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Foreign tables", - "Objects": [], - "FeatureDescription": "During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.", - "MinimumVersionsFixedIn": null - }, - { - "FeatureName": "Policies", - "Objects": [], - "FeatureDescription": "There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.", - "MinimumVersionsFixedIn": null - } - ], + "MigrationCaveats": null, "UnsupportedQueryConstructs": null, "UnsupportedPlPgSqlObjects": null } diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 21e006c08..f67697ae1 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -984,9 +984,9 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures := make([]UnsupportedFeature, 0) for _, indexMethod := range queryissue.UnsupportedIndexMethods { displayIndexMethod := strings.ToUpper(indexMethod) - feature := fmt.Sprintf("%s indexes", displayIndexMethod) + featureName := fmt.Sprintf("%s indexes", displayIndexMethod) reason := fmt.Sprintf(INDEX_METHOD_ISSUE_REASON, displayIndexMethod) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(feature, reason, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(featureName, reason, "", schemaAnalysisReport, false, "")) } unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CONSTRAINT_TRIGGERS_FEATURE, CONSTRAINT_TRIGGER_ISSUE_REASON, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(INHERITED_TABLES_FEATURE, INHERITANCE_ISSUE_REASON, "", schemaAnalysisReport, false, "")) @@ -1015,7 +1015,9 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.LARGE_OBJECT_FUNCTIONS_NAME, queryissue.LARGE_OBJECT_FUNCTIONS_NAME, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REGEX_FUNCTIONS_FEATURE, "", queryissue.REGEX_FUNCTIONS, schemaAnalysisReport, false, "")) - return unsupportedFeatures, nil + return lo.Filter(unsupportedFeatures, func(f UnsupportedFeature, _ int) bool { + return len(f.Objects) > 0 + }), nil } func getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisiReport utils.SchemaReport, unsupportedIndexDatatypes []string) UnsupportedFeature { @@ -1038,7 +1040,6 @@ func getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisiReport utils.Schem } } } - return indexesOnComplexTypesFeature } @@ -1046,7 +1047,9 @@ func fetchUnsupportedOracleFeaturesFromSchemaReport(schemaAnalysisReport utils.S log.Infof("fetching unsupported features for Oracle...") unsupportedFeatures := make([]UnsupportedFeature, 0) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(COMPOUND_TRIGGER_FEATURE, COMPOUND_TRIGGER_ISSUE_REASON, "", schemaAnalysisReport, false, "")) - return unsupportedFeatures, nil + return lo.Filter(unsupportedFeatures, func(f UnsupportedFeature, _ int) bool { + return len(f.Objects) > 0 + }), nil } var OracleUnsupportedIndexTypes = []string{"CLUSTER INDEX", "DOMAIN INDEX", "FUNCTION-BASED DOMAIN INDEX", "IOT - TOP INDEX", "NORMAL/REV INDEX", "FUNCTION-BASED NORMAL/REV INDEX"} @@ -1095,7 +1098,9 @@ func fetchUnsupportedObjectTypes() ([]UnsupportedFeature, error) { unsupportedFeatures = append(unsupportedFeatures, UnsupportedFeature{VIRTUAL_COLUMNS_FEATURE, virtualColumns, false, "", "", nil}) unsupportedFeatures = append(unsupportedFeatures, UnsupportedFeature{INHERITED_TYPES_FEATURE, inheritedTypes, false, "", "", nil}) unsupportedFeatures = append(unsupportedFeatures, UnsupportedFeature{UNSUPPORTED_PARTITIONING_METHODS_FEATURE, unsupportedPartitionTypes, false, "", "", nil}) - return unsupportedFeatures, nil + return lo.Filter(unsupportedFeatures, func(f UnsupportedFeature, _ int) bool { + return len(f.Objects) > 0 + }), nil } func fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport utils.SchemaReport) []UnsupportedFeature { @@ -1397,16 +1402,26 @@ func addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration for _, col := range unsupportedDataTypesForLiveMigration { columns = append(columns, ObjectInfo{ObjectName: fmt.Sprintf("%s.%s.%s (%s)", col.SchemaName, col.TableName, col.ColumnName, col.DataType)}) } - migrationCaveats = append(migrationCaveats, UnsupportedFeature{UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE, columns, false, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_ISSUE, nil}) + if len(columns) > 0 { + migrationCaveats = append(migrationCaveats, UnsupportedFeature{UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE, columns, false, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_ISSUE, nil}) + } } if len(unsupportedDataTypesForLiveMigrationWithFForFB) > 0 { columns := make([]ObjectInfo, 0) for _, col := range unsupportedDataTypesForLiveMigrationWithFForFB { columns = append(columns, ObjectInfo{ObjectName: fmt.Sprintf("%s.%s.%s (%s)", col.SchemaName, col.TableName, col.ColumnName, col.DataType)}) } - migrationCaveats = append(migrationCaveats, UnsupportedFeature{UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE, columns, false, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_ISSUE, nil}) + if len(columns) > 0 { + migrationCaveats = append(migrationCaveats, UnsupportedFeature{UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE, columns, false, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_ISSUE, nil}) + } } - assessmentReport.MigrationCaveats = migrationCaveats + migrationCaveats = lo.Filter(migrationCaveats, func(m UnsupportedFeature, _ int) bool { + return len(m.Objects) > 0 + }) + if len(migrationCaveats) > 0 { + assessmentReport.MigrationCaveats = migrationCaveats + } + } } From 81c30b04af198fedf5585d105c44a4f6ba82bdcf Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Mon, 23 Dec 2024 19:10:37 +0530 Subject: [PATCH 04/17] Add --skip-debezium flag in installer script to avoid installing debezium when only offline setup is required (#2109) --- installer_scripts/install-yb-voyager | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/installer_scripts/install-yb-voyager b/installer_scripts/install-yb-voyager index f026999f5..912f3e069 100755 --- a/installer_scripts/install-yb-voyager +++ b/installer_scripts/install-yb-voyager @@ -12,6 +12,7 @@ LOG_FILE=/tmp/install-yb-voyager.log VERSION="latest" ONLY_PG="false" +SKIP_DEBEZIUM="false" trap on_exit EXIT @@ -323,14 +324,18 @@ check_java() { } install_debezium_server(){ + if [ $SKIP_DEBEZIUM == "true" ] ; then + return + fi + if [ "${VERSION}" != "local" ] then output "Installing debezium:${DEBEZIUM_VERSION}" install_debezium_server_from_release return fi - output "Installing debezium:${VERSION}." + output "Installing debezium:${VERSION}." check_install_maven clean_debezium install_debezium_local ${DEBEZIUM_LOCAL_REF} @@ -538,9 +543,9 @@ rebuild_voyager_local() { get_passed_options() { if [ "$1" == "linux" ] then - OPTS=$(getopt -o "lpvV", --long install-from-local-source,only-pg-support,rebuild-voyager-local,version: --name 'install-yb-voyager' -- $ARGS_LINUX) + OPTS=$(getopt -o "lpdvV", --long install-from-local-source,only-pg-support,skip-debezium,rebuild-voyager-local,version: --name 'install-yb-voyager' -- $ARGS_LINUX) else - OPTS=$(getopt lpvV $ARGS_MACOS) + OPTS=$(getopt lpdvV $ARGS_MACOS) fi eval set -- "$OPTS" @@ -555,6 +560,10 @@ get_passed_options() { ONLY_PG="true"; shift ;; + -d | --skip-debezium ) + SKIP_DEBEZIUM="true" + shift + ;; -v | --rebuild-voyager-local ) REBUILD_VOYAGER_LOCAL="true"; shift From 23033f8a1ed5bf6c2c5f1b85c9f663e0a6561425 Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Tue, 24 Dec 2024 16:22:26 +0530 Subject: [PATCH 05/17] Fixed the logic for checking source db version compatibility and added it to assess migration too for PG (#2115) --- yb-voyager/cmd/assessMigrationCommand.go | 8 ++++++++ yb-voyager/src/srcdb/postgres.go | 16 +++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index f67697ae1..0adcff8fe 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -335,6 +335,14 @@ func assessMigration() (err error) { // We will require source db connection for the below checks // Check if required binaries are installed. if source.RunGuardrailsChecks { + // Check source database version. + log.Info("checking source DB version") + err = source.DB().CheckSourceDBVersion(exportType) + if err != nil { + return fmt.Errorf("source DB version check failed: %w", err) + } + + // Check if required binaries are installed. binaryCheckIssues, err := checkDependenciesForExport() if err != nil { return fmt.Errorf("failed to check dependencies for assess migration: %w", err) diff --git a/yb-voyager/src/srcdb/postgres.go b/yb-voyager/src/srcdb/postgres.go index 34249fafd..4c90ffb5e 100644 --- a/yb-voyager/src/srcdb/postgres.go +++ b/yb-voyager/src/srcdb/postgres.go @@ -42,7 +42,7 @@ import ( const MIN_SUPPORTED_PG_VERSION_OFFLINE = "9" const MIN_SUPPORTED_PG_VERSION_LIVE = "10" -const MAX_SUPPORTED_PG_VERSION = "16" +const MAX_SUPPORTED_PG_VERSION = "17" const MISSING = "MISSING" const GRANTED = "GRANTED" const NO_USAGE_PERMISSION = "NO USAGE PERMISSION" @@ -979,14 +979,24 @@ func (pg *PostgreSQL) CheckSourceDBVersion(exportType string) error { if pgVersion == "" { return fmt.Errorf("failed to get source database version") } + + // Extract the major version from the full version string + // Version can be like: 17.2 (Ubuntu 17.2-1.pgdg20.04+1), 17.2, 17alpha1 etc. + re := regexp.MustCompile(`^(\d+)`) + match := re.FindStringSubmatch(pgVersion) + if len(match) < 2 { + return fmt.Errorf("failed to extract major version from source database version: %s", pgVersion) + } + majorVersion := match[1] + supportedVersionRange := fmt.Sprintf("%s to %s", MIN_SUPPORTED_PG_VERSION_OFFLINE, MAX_SUPPORTED_PG_VERSION) - if version.CompareSimple(pgVersion, MAX_SUPPORTED_PG_VERSION) > 0 || version.CompareSimple(pgVersion, MIN_SUPPORTED_PG_VERSION_OFFLINE) < 0 { + if version.CompareSimple(majorVersion, MAX_SUPPORTED_PG_VERSION) > 0 || version.CompareSimple(majorVersion, MIN_SUPPORTED_PG_VERSION_OFFLINE) < 0 { return fmt.Errorf("current source db version: %s. Supported versions: %s", pgVersion, supportedVersionRange) } // for live migration if exportType == utils.CHANGES_ONLY || exportType == utils.SNAPSHOT_AND_CHANGES { - if version.CompareSimple(pgVersion, MIN_SUPPORTED_PG_VERSION_LIVE) < 0 { + if version.CompareSimple(majorVersion, MIN_SUPPORTED_PG_VERSION_LIVE) < 0 { supportedVersionRange = fmt.Sprintf("%s to %s", MIN_SUPPORTED_PG_VERSION_LIVE, MAX_SUPPORTED_PG_VERSION) utils.PrintAndLog(color.RedString("Warning: Live Migration: Current source db version: %s. Supported versions: %s", pgVersion, supportedVersionRange)) } From 521ac3147ab375e241c8c05fc60bbec0bd9f96c5 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Tue, 24 Dec 2024 16:59:58 +0530 Subject: [PATCH 06/17] [DB-14231] Report Security Invoker View in analyze and assess migration commands generated reports (#2108) - Implement ViewProcessor's `Process()` method using protomsg approach. - Made use of existing `TraverseParseTree()` function to write a helper func TraverseAndExtractDefNameFromDefElem()` to extract all defnames in any type of DDL. * Added unit tests * Added sql in end to end test for security invoker views --- migtests/scripts/functions.sh | 2 +- .../expectedAssessmentReport.json | 43 +++++++++++++---- .../tests/pg/assessment-report-test/init-db | 1 + .../pg_assessment_report.sql | 22 +++++++++ yb-voyager/cmd/analyzeSchema.go | 1 + yb-voyager/cmd/assessMigrationCommand.go | 2 + yb-voyager/src/query/queryissue/constants.go | 3 ++ .../src/query/queryissue/detectors_ddl.go | 13 ++++- yb-voyager/src/query/queryissue/issues_ddl.go | 16 ++++++- .../src/query/queryissue/issues_ddl_test.go | 24 ++++++++++ .../queryissue/parser_issue_detector_test.go | 28 ++++++----- .../src/query/queryparser/ddl_processor.go | 32 +++++++++++-- .../src/query/queryparser/helpers_protomsg.go | 48 +++++++++++++++++++ .../src/query/queryparser/traversal_proto.go | 3 ++ 14 files changed, 209 insertions(+), 29 deletions(-) diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index eeab23678..18f3ff9a2 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -932,7 +932,7 @@ compare_files() { return 0 else echo "Data does not match expected report." - diff_output=$(diff "$file1" "$file2") + diff_output=$(diff --context "$file1" "$file2") echo "$diff_output" return 1 fi diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index edfe82ecd..1057ad9fd 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -38,15 +38,15 @@ }, { "ObjectType": "SEQUENCE", - "TotalCount": 27, + "TotalCount": 28, "InvalidCount": 0, - "ObjectNames": "public.ordersentry_order_id_seq, public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.employees2_id_seq, public.ext_test_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.orders2_id_seq, public.parent_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" + "ObjectNames": "public.ordersentry_order_id_seq, public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.employees_employee_id_seq, public.employees2_id_seq, public.ext_test_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.orders2_id_seq, public.parent_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" }, { "ObjectType": "TABLE", - "TotalCount": 66, + "TotalCount": 67, "InvalidCount": 23, - "ObjectNames": "public.ordersentry, public.library_nested, public.orders_lateral, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.sales_region, public.boston, public.c, public.parent_table, public.child_table, public.citext_type, public.combined_tbl, public.documents, public.employees2, public.ext_test, public.foo, public.inet_type, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.orders, public.orders2, public.products, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.sales_region, schema2.boston, schema2.c, schema2.parent_table, schema2.child_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.orders, schema2.orders2, schema2.products, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" + "ObjectNames": "public.ordersentry, public.library_nested, public.orders_lateral, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.sales_region, public.boston, public.c, public.parent_table, public.child_table, public.citext_type, public.combined_tbl, public.documents, public.employees2, public.ext_test, public.foo, public.inet_type, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.orders, public.orders2, public.products, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.sales_region, schema2.boston, schema2.c, schema2.parent_table, schema2.child_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.orders, schema2.orders2, schema2.products, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2, public.employees" }, { "ObjectType": "INDEX", @@ -73,9 +73,9 @@ }, { "ObjectType": "VIEW", - "TotalCount": 7, - "InvalidCount": 3, - "ObjectNames": "public.ordersentry_view, public.sales_employees, schema2.sales_employees, test_views.v1, test_views.v2, test_views.v3, test_views.v4" + "TotalCount": 8, + "InvalidCount": 4, + "ObjectNames": "public.ordersentry_view, public.sales_employees, schema2.sales_employees, test_views.v1, test_views.v2, test_views.v3, test_views.v4, public.view_explicit_security_invoker" }, { "ObjectType": "TRIGGER", @@ -169,9 +169,10 @@ "test_views.abc_mview", "test_views.view_table1", "public.library_nested", - "public.orders_lateral" + "public.orders_lateral", + "public.employees" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 72 objects (64 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 73 objects (65 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": [ "public.combined_tbl", "public.citext_type", @@ -555,6 +556,16 @@ } ], "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Security Invoker Views", + "Objects": [ + { + "ObjectName": "public.view_explicit_security_invoker", + "SqlStatement": "CREATE VIEW public.view_explicit_security_invoker WITH (security_invoker='true') AS\n SELECT employees.employee_id,\n employees.first_name\n FROM public.employees;" + } + ], + "MinimumVersionsFixedIn": null } ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", @@ -839,6 +850,20 @@ "ParentTableName": null, "SizeInBytes": 8192 }, + { + "SchemaName": "public", + "ObjectName": "employees", + "RowCount": 5, + "ColumnCount": 4, + "Reads": 0, + "Writes": 10, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 8192 + }, { "SchemaName": "public", "ObjectName": "employees2", diff --git a/migtests/tests/pg/assessment-report-test/init-db b/migtests/tests/pg/assessment-report-test/init-db index c4016a4c7..516009448 100755 --- a/migtests/tests/pg/assessment-report-test/init-db +++ b/migtests/tests/pg/assessment-report-test/init-db @@ -25,6 +25,7 @@ cat < "$TEMP_SQL" \i ${TEST_DIR}/../views-and-rules/pg_views_and_rules_automation.sql; \i pg_assessment_report.sql; \i unsupported_query_constructs.sql; + CREATE SCHEMA IF NOT EXISTS schema2; SET SEARCH_PATH TO schema2; \i ${TEST_DIR}/../misc-objects-1/schema.sql; diff --git a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql index 0c089922b..c23e2ddc5 100644 --- a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql +++ b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql @@ -362,3 +362,25 @@ BEGIN END IF; END; $$ LANGUAGE plpgsql; + +-- SECURITY INVOKER VIEW +CREATE TABLE public.employees ( + employee_id SERIAL PRIMARY KEY, + first_name VARCHAR(100), + last_name VARCHAR(100), + department VARCHAR(50) +); + +INSERT INTO public.employees (first_name, last_name, department) +VALUES + ('Alice', 'Smith', 'HR'), + ('Bob', 'Jones', 'Finance'), + ('Charlie', 'Brown', 'IT'), + ('Diana', 'Prince', 'HR'), + ('Ethan', 'Hunt', 'Security'); + +CREATE VIEW public.view_explicit_security_invoker +WITH (security_invoker = true) AS + SELECT employee_id, first_name + FROM public.employees; + diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 85d1ce25c..a4dda2b03 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -225,6 +225,7 @@ const ( INDEX_METHOD_ISSUE_REASON = "Schema contains %s index which is not supported." INSUFFICIENT_COLUMNS_IN_PK_FOR_PARTITION = "insufficient columns in the PRIMARY KEY constraint definition in CREATE TABLE" GIN_INDEX_DETAILS = "There are some GIN indexes present in the schema, but GIN indexes are partially supported in YugabyteDB as mentioned in (https://github.com/yugabyte/yugabyte-db/issues/7850) so take a look and modify them if not supported." + SECURITY_INVOKER_VIEWS_ISSUE = "Security Invoker Views not supported yet" ) // Reports one case in JSON diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 0adcff8fe..945c42e9a 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -990,6 +990,7 @@ func getUnsupportedFeaturesFromSchemaAnalysisReport(featureName string, issueRea func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.SchemaReport) ([]UnsupportedFeature, error) { log.Infof("fetching unsupported features for PG...") unsupportedFeatures := make([]UnsupportedFeature, 0) + for _, indexMethod := range queryissue.UnsupportedIndexMethods { displayIndexMethod := strings.ToUpper(indexMethod) featureName := fmt.Sprintf("%s indexes", displayIndexMethod) @@ -1022,6 +1023,7 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SYSTEM_COLUMNS_NAME, queryissue.SYSTEM_COLUMNS_NAME, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.LARGE_OBJECT_FUNCTIONS_NAME, queryissue.LARGE_OBJECT_FUNCTIONS_NAME, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REGEX_FUNCTIONS_FEATURE, "", queryissue.REGEX_FUNCTIONS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SECURITY_INVOKER_VIEWS_NAME, "", queryissue.SECURITY_INVOKER_VIEWS, schemaAnalysisReport, false, "")) return lo.Filter(unsupportedFeatures, func(f UnsupportedFeature, _ int) bool { return len(f.Objects) > 0 diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index 3a6d42453..af2d24a6c 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -53,6 +53,9 @@ const ( LARGE_OBJECT_FUNCTIONS = "LARGE_OBJECT_FUNCTIONS" LARGE_OBJECT_FUNCTIONS_NAME = "Large Object Functions" + SECURITY_INVOKER_VIEWS = "SECURITY_INVOKER_VIEWS" + SECURITY_INVOKER_VIEWS_NAME = "Security Invoker Views" + ADVISORY_LOCKS = "ADVISORY_LOCKS" SYSTEM_COLUMNS = "SYSTEM_COLUMNS" XML_FUNCTIONS = "XML_FUNCTIONS" diff --git a/yb-voyager/src/query/queryissue/detectors_ddl.go b/yb-voyager/src/query/queryissue/detectors_ddl.go index e14da93bf..532ee2dc2 100644 --- a/yb-voyager/src/query/queryissue/detectors_ddl.go +++ b/yb-voyager/src/query/queryissue/detectors_ddl.go @@ -545,7 +545,7 @@ func (tid *TriggerIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Quer } if unsupportedLargeObjectFunctions.ContainsOne(trigger.FuncName) { - //Can't detect trigger func name using the genericIssues's FuncCallDetector + //Can't detect trigger func name using the genericIssues's FuncCallDetector //as trigger execute Func name is not a FuncCall node, its []pg_query.Node issues = append(issues, NewLOFuntionsIssue( obj.GetObjectType(), @@ -562,7 +562,16 @@ func (tid *TriggerIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Quer type ViewIssueDetector struct{} func (v *ViewIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]QueryIssue, error) { - return nil, nil + view, ok := obj.(*queryparser.View) + if !ok { + return nil, fmt.Errorf("invalid object type: expected View") + } + var issues []QueryIssue + + if view.SecurityInvoker { + issues = append(issues, NewSecurityInvokerViewIssue(obj.GetObjectType(), obj.GetObjectName(), "")) + } + return issues, nil } // ==============MVIEW ISSUE DETECTOR ====================== diff --git a/yb-voyager/src/query/queryissue/issues_ddl.go b/yb-voyager/src/query/queryissue/issues_ddl.go index 65505d9d0..914dcd536 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl.go +++ b/yb-voyager/src/query/queryissue/issues_ddl.go @@ -431,11 +431,23 @@ var loDatatypeIssue = issue.Issue{ TypeName: "Unsupported datatype - lo", Suggestion: "Large objects are not yet supported in YugabyteDB, no workaround available currently", GH: "https://github.com/yugabyte/yugabyte-db/issues/25318", - DocsLink: "", //TODO + DocsLink: "", // TODO } func NewLODatatypeIssue(objectType string, objectName string, SqlStatement string, colName string) QueryIssue { issue := loDatatypeIssue issue.TypeName = fmt.Sprintf("%s on column - %s", issue.TypeName, colName) return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) -} \ No newline at end of file +} + +var securityInvokerViewIssue = issue.Issue{ + Type: SECURITY_INVOKER_VIEWS, + TypeName: "Security Invoker Views not supported yet", + Suggestion: "Security Invoker Views are not yet supported in YugabyteDB, no workaround available currently", + GH: "", // TODO + DocsLink: "", // TODO +} + +func NewSecurityInvokerViewIssue(objectType string, objectName string, SqlStatement string) QueryIssue { + return newQueryIssue(securityInvokerViewIssue, objectType, objectName, SqlStatement, map[string]interface{}{}) +} diff --git a/yb-voyager/src/query/queryissue/issues_ddl_test.go b/yb-voyager/src/query/queryissue/issues_ddl_test.go index d4d6fac32..68993779a 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl_test.go +++ b/yb-voyager/src/query/queryissue/issues_ddl_test.go @@ -216,6 +216,28 @@ func testLoDatatypeIssue(t *testing.T) { assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "does not exist", loDatatypeIssue) } +func testSecurityInvokerView(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, ` + CREATE TABLE public.employees ( + employee_id SERIAL PRIMARY KEY, + first_name VARCHAR(100), + last_name VARCHAR(100), + department VARCHAR(50) + ); + + CREATE VIEW public.view_explicit_security_invoker + WITH (security_invoker = true) AS + SELECT employee_id, first_name + FROM public.employees;`) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "unrecognized parameter", securityInvokerViewIssue) +} + func TestDDLIssuesInYBVersion(t *testing.T) { var err error ybVersion := os.Getenv("YB_VERSION") @@ -269,4 +291,6 @@ func TestDDLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "lo datatype", ybVersion), testLoDatatypeIssue) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "security invoker view", ybVersion), testSecurityInvokerView) + assert.True(t, success) } diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index e7c979118..066d738fb 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -156,9 +156,13 @@ ADD CONSTRAINT valid_invoice_structure CHECK (xpath_exists('/invoice/customer', data));` stmt18 = `CREATE INDEX idx_invoices on invoices (xpath('/invoice/customer/text()', data));` stmt19 = `create table test_lo_default (id int, raster lo DEFAULT lo_import('3242'));` + stmt20 = `CREATE VIEW public.view_explicit_security_invoker + WITH (security_invoker = true) AS + SELECT employee_id, first_name + FROM public.employees;` ) -func modifyiedIssuesforPLPGSQL(issues []QueryIssue, objType string, objName string) []QueryIssue { +func modifiedIssuesforPLPGSQL(issues []QueryIssue, objType string, objName string) []QueryIssue { return lo.Map(issues, func(i QueryIssue, _ int) QueryIssue { i.ObjectType = objType i.ObjectName = objName @@ -221,9 +225,9 @@ func TestAllIssues(t *testing.T) { } //Should modify it in similar way we do it actual code as the particular DDL issue in plpgsql can have different Details map on the basis of objectType - stmtsWithExpectedIssues[stmt1] = modifyiedIssuesforPLPGSQL(stmtsWithExpectedIssues[stmt1], "FUNCTION", "list_high_earners") + stmtsWithExpectedIssues[stmt1] = modifiedIssuesforPLPGSQL(stmtsWithExpectedIssues[stmt1], "FUNCTION", "list_high_earners") - stmtsWithExpectedIssues[stmt2] = modifyiedIssuesforPLPGSQL(stmtsWithExpectedIssues[stmt2], "FUNCTION", "process_order") + stmtsWithExpectedIssues[stmt2] = modifiedIssuesforPLPGSQL(stmtsWithExpectedIssues[stmt2], "FUNCTION", "process_order") for _, stmt := range requiredDDLs { err := parserIssueDetector.ParseRequiredDDLs(stmt) @@ -276,6 +280,9 @@ func TestDDLIssues(t *testing.T) { NewLODatatypeIssue("TABLE", "test_lo_default", stmt19, "raster"), NewLOFuntionsIssue("TABLE", "test_lo_default", stmt19), }, + stmt20: []QueryIssue{ + NewSecurityInvokerViewIssue("VIEW", "public.view_explicit_security_invoker", stmt20), + }, } for _, stmt := range requiredDDLs { err := parserIssueDetector.ParseRequiredDDLs(stmt) @@ -415,7 +422,7 @@ BEGIN END; $$ LANGUAGE plpgsql; `, -`CREATE TRIGGER t_raster BEFORE UPDATE OR DELETE ON image + `CREATE TRIGGER t_raster BEFORE UPDATE OR DELETE ON image FOR EACH ROW EXECUTE FUNCTION lo_manage(raster);`, } @@ -442,12 +449,11 @@ $$ LANGUAGE plpgsql; NewLOFuntionsIssue("TRIGGER", "t_raster ON image", sqls[5]), }, } - expectedSQLsWithIssues[sqls[0]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[0]], "FUNCTION", "manage_large_object") - expectedSQLsWithIssues[sqls[1]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[1]], "FUNCTION", "import_file_to_table") - expectedSQLsWithIssues[sqls[2]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[2]], "FUNCTION", "export_large_object") - expectedSQLsWithIssues[sqls[3]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[3]], "PROCEDURE", "read_large_object") - expectedSQLsWithIssues[sqls[4]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[4]], "FUNCTION", "write_to_large_object") - + expectedSQLsWithIssues[sqls[0]] = modifiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[0]], "FUNCTION", "manage_large_object") + expectedSQLsWithIssues[sqls[1]] = modifiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[1]], "FUNCTION", "import_file_to_table") + expectedSQLsWithIssues[sqls[2]] = modifiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[2]], "FUNCTION", "export_large_object") + expectedSQLsWithIssues[sqls[3]] = modifiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[3]], "PROCEDURE", "read_large_object") + expectedSQLsWithIssues[sqls[4]] = modifiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[4]], "FUNCTION", "write_to_large_object") parserIssueDetector := NewParserIssueDetector() @@ -456,7 +462,6 @@ $$ LANGUAGE plpgsql; fmt.Printf("%v", issues) assert.NoError(t, err, "Error detecting issues for statement: %s", stmt) - assert.Equal(t, len(expectedIssues), len(issues), "Mismatch in issue count for statement: %s", stmt) for _, expectedIssue := range expectedIssues { found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { @@ -466,6 +471,7 @@ $$ LANGUAGE plpgsql; } } } + // currently, both FuncCallDetector and XmlExprDetector can detect XMLFunctionsIssue // statement below has both XML functions and XML expressions. // but we want to only return one XMLFunctionsIssue from parserIssueDetector.getDMLIssues diff --git a/yb-voyager/src/query/queryparser/ddl_processor.go b/yb-voyager/src/query/queryparser/ddl_processor.go index 2d2a6e766..a14485cd8 100644 --- a/yb-voyager/src/query/queryparser/ddl_processor.go +++ b/yb-voyager/src/query/queryparser/ddl_processor.go @@ -22,6 +22,7 @@ import ( pg_query "github.com/pganalyze/pg_query_go/v6" "github.com/samber/lo" + log "github.com/sirupsen/logrus" ) // Base parser interface @@ -849,16 +850,39 @@ func (v *ViewProcessor) Process(parseTree *pg_query.ParseResult) (DDLObject, err if !ok { return nil, fmt.Errorf("not a CREATE VIEW statement") } + + viewSchemaName := viewNode.ViewStmt.View.Schemaname + viewName := viewNode.ViewStmt.View.Relname + qualifiedViewName := lo.Ternary(viewSchemaName == "", viewName, viewSchemaName+"."+viewName) + + /* + view_stmt:{view:{schemaname:"public" relname:"invoker_view" inh:true relpersistence:"p" location:12} + query:{select_stmt:{target_list:{res_target:{val:{column_ref:{fields:{string:{sval:"id"}} location:95}} location:95}} + from_clause:{...} + where_clause:{...} + options:{def_elem:{defname:"security_invoker" arg:{string:{sval:"true"}} defaction:DEFELEM_UNSPEC location:32}} + options:{def_elem:{defname:"security_barrier" arg:{string:{sval:"false"}} defaction:DEFELEM_UNSPEC location:57}} + with_check_option:NO_CHECK_OPTION} + */ + log.Infof("checking the view '%s' is security invoker view", qualifiedViewName) + msg := GetProtoMessageFromParseTree(parseTree) + defNames, err := TraverseAndExtractDefNamesFromDefElem(msg) + if err != nil { + return nil, err + } + view := View{ - SchemaName: viewNode.ViewStmt.View.Schemaname, - ViewName: viewNode.ViewStmt.View.Relname, + SchemaName: viewSchemaName, + ViewName: viewName, + SecurityInvoker: slices.Contains(defNames, "security_invoker"), } return &view, nil } type View struct { - SchemaName string - ViewName string + SchemaName string + ViewName string + SecurityInvoker bool } func (v *View) GetObjectName() string { diff --git a/yb-voyager/src/query/queryparser/helpers_protomsg.go b/yb-voyager/src/query/queryparser/helpers_protomsg.go index 0eb609d15..c79aebc8c 100644 --- a/yb-voyager/src/query/queryparser/helpers_protomsg.go +++ b/yb-voyager/src/query/queryparser/helpers_protomsg.go @@ -16,6 +16,7 @@ limitations under the License. package queryparser import ( + "fmt" "strings" pg_query "github.com/pganalyze/pg_query_go/v6" @@ -328,6 +329,21 @@ func GetStatementType(msg protoreflect.Message) string { return GetMsgFullName(node) } +func getOneofActiveNode(msg protoreflect.Message) protoreflect.Message { + nodeField := getOneofActiveField(msg, "node") + if nodeField == nil { + return nil + } + + value := msg.Get(nodeField) + node := value.Message() + if node == nil || !node.IsValid() { + return nil + } + + return node +} + // == Generic helper functions == // GetStringField retrieves a string field from a message. @@ -370,3 +386,35 @@ func GetSchemaAndObjectName(nameList protoreflect.List) (string, string) { } return schemaName, objectName } + +/* +Example: +options:{def_elem:{defname:"security_invoker" arg:{string:{sval:"true"}} defaction:DEFELEM_UNSPEC location:32}} +options:{def_elem:{defname:"security_barrier" arg:{string:{sval:"false"}} defaction:DEFELEM_UNSPEC location:57}} + +Extract all defnames from the def_eleme node +*/ +func TraverseAndExtractDefNamesFromDefElem(msg protoreflect.Message) ([]string, error) { + var defNames []string + + collectorFunc := func(msg protoreflect.Message) error { + if GetMsgFullName(msg) != PG_QUERY_DEFELEM_NODE { + return nil + } + + defName := GetStringField(msg, "defname") + // TODO(future): + // defValNode = GetMessageField(msg, "arg") + // defVal = GetStringField(defValNode, "sval") + + defNames = append(defNames, defName) + return nil + } + visited := make(map[protoreflect.Message]bool) + err := TraverseParseTree(msg, visited, collectorFunc) + if err != nil { + return nil, fmt.Errorf("failed to traverse parse tree for fetching defnames: %w", err) + } + + return defNames, nil +} diff --git a/yb-voyager/src/query/queryparser/traversal_proto.go b/yb-voyager/src/query/queryparser/traversal_proto.go index 3ed84f4ab..d54aefe0e 100644 --- a/yb-voyager/src/query/queryparser/traversal_proto.go +++ b/yb-voyager/src/query/queryparser/traversal_proto.go @@ -36,10 +36,13 @@ const ( PG_QUERY_RANGEVAR_NODE = "pg_query.RangeVar" PG_QUERY_RANGETABLEFUNC_NODE = "pg_query.RangeTableFunc" PG_QUERY_PARAMREF_NODE = "pg_query.ParamRef" + PG_QUERY_DEFELEM_NODE = "pg_query.DefElem" PG_QUERY_INSERTSTMT_NODE = "pg_query.InsertStmt" PG_QUERY_UPDATESTMT_NODE = "pg_query.UpdateStmt" PG_QUERY_DELETESTMT_NODE = "pg_query.DeleteStmt" + + PG_QUERY_VIEWSTMT_NODE = "pg_query.ViewStmt" ) // function type for processing nodes during traversal From 89c4f76a52ec838e74e7c1ae6df7de70e076dff0 Mon Sep 17 00:00:00 2001 From: Shaharuk Shaikh <56402576+shaharuk-yb@users.noreply.github.com> Date: Tue, 24 Dec 2024 18:35:47 +0530 Subject: [PATCH 07/17] [Sizing Calc] Consider specific object types, skip considering primary key object type in sizing calculator workflow (#2089) * skip considering primary key object type in sizing calculator workflow --- yb-voyager/src/migassessment/sizing.go | 18 +++++++++++++++++- yb-voyager/src/migassessment/sizing_test.go | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/yb-voyager/src/migassessment/sizing.go b/yb-voyager/src/migassessment/sizing.go index 1d45a1167..05383341a 100644 --- a/yb-voyager/src/migassessment/sizing.go +++ b/yb-voyager/src/migassessment/sizing.go @@ -25,6 +25,7 @@ import ( "net/http" "os" "path/filepath" + "strings" "github.com/samber/lo" @@ -139,6 +140,12 @@ func getExperimentDBPath() string { //go:embed resources/yb_2024_0_source.db var experimentData20240 []byte +var SourceMetadataObjectTypesToUse = []string{ + "%table%", + "%index%", + "materialized view", +} + func SizingAssessment() error { log.Infof("loading metadata files for sharding assessment") @@ -1228,6 +1235,14 @@ Returns: float64: The total size of the source database in gigabytes. */ func getSourceMetadata(sourceDB *sql.DB) ([]SourceDBMetadata, []SourceDBMetadata, float64, error) { + // Construct the WHERE clause dynamically using LIKE + var likeConditions []string + for _, pattern := range SourceMetadataObjectTypesToUse { + likeConditions = append(likeConditions, fmt.Sprintf("object_type LIKE '%s'", pattern)) + } + // Join the LIKE conditions with OR + whereClause := strings.Join(likeConditions, " OR ") + query := fmt.Sprintf(` SELECT schema_name, object_name, @@ -1239,8 +1254,9 @@ func getSourceMetadata(sourceDB *sql.DB) ([]SourceDBMetadata, []SourceDBMetadata size_in_bytes, column_count FROM %v + WHERE %s ORDER BY IFNULL(size_in_bytes, 0) ASC - `, GetTableIndexStatName()) + `, GetTableIndexStatName(), whereClause) rows, err := sourceDB.Query(query) if err != nil { return nil, nil, 0.0, fmt.Errorf("failed to query source metadata with query [%s]: %w", query, err) diff --git a/yb-voyager/src/migassessment/sizing_test.go b/yb-voyager/src/migassessment/sizing_test.go index bdf28050f..dff6c5252 100644 --- a/yb-voyager/src/migassessment/sizing_test.go +++ b/yb-voyager/src/migassessment/sizing_test.go @@ -27,7 +27,7 @@ import ( "github.com/stretchr/testify/assert" ) -var AssessmentDbSelectQuery = fmt.Sprintf("(?i)SELECT schema_name,.* FROM %v ORDER BY .* ASC", TABLE_INDEX_STATS) +var AssessmentDbSelectQuery = fmt.Sprintf("(?i)SELECT schema_name,.* FROM %v .* ORDER BY .* ASC", TABLE_INDEX_STATS) var AssessmentDBColumns = []string{"schema_name", "object_name", "row_count", "reads_per_second", "writes_per_second", "is_index", "parent_table_name", "size_in_bytes", "column_count"} From 3c45ffc29061afd0d996ddbcbbd07bef5b734d14 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Tue, 24 Dec 2024 18:47:51 +0530 Subject: [PATCH 08/17] Reporting Json Constructor / Query Functions and AnyValue Aggregate functions (#2107) 1. Added support to report the JSON constructor functions (`JSON_OBJECT`, `JSON_OBJECTAGG`, `JSON_ARRAYAGG`, `JSON_ARRAY`) and JSON query functions (`JSON_QUERY`, `JSON_EXISTS`, `JSON_VALUE`) with new detectors as these all have a separate to detect. Currently, the code and unit tests are added for reporting the `JSON_TABLE` in DMLs but it can't be reported in the Query constructs sections as PGSS is giving the queries with placeholders ($1, $2...) and `JSON_TABLE` syntax doesn't allow it so the parser is failing with that. https://yugabyte.atlassian.net/browse/DB-14542 2. Added support to report the aggregate functions, reporting the `ANY_VALUE` agg function in this PR. https://yugabyte.atlassian.net/browse/DB-14220 3. Fixed the assessment template for adding the Supported Versions in HTML for PL/pgSQL and Migration Caveats sections. --- .../dummy-export-dir/schema/views/view.sql | 13 +- .../tests/analyze-schema/expected_issues.json | 30 ++++ migtests/tests/analyze-schema/summary.json | 6 +- .../pg_assessment_report_uqc.sql | 6 +- .../unsupported_query_constructs.sql | 7 +- yb-voyager/cmd/assessMigrationCommand.go | 3 + .../migration_assessment_report.template | 22 ++- yb-voyager/src/query/queryissue/constants.go | 13 +- yb-voyager/src/query/queryissue/detectors.go | 112 +++++++++++++- .../src/query/queryissue/detectors_ddl.go | 1 + .../src/query/queryissue/detectors_test.go | 123 +++++++-------- yb-voyager/src/query/queryissue/helpers.go | 18 +++ yb-voyager/src/query/queryissue/issues_dml.go | 65 +++++++- .../src/query/queryissue/issues_dml_test.go | 63 +++++++- .../query/queryissue/parser_issue_detector.go | 2 + .../queryissue/parser_issue_detector_test.go | 146 ++++++++++++++++-- .../src/query/queryparser/helpers_protomsg.go | 12 ++ .../src/query/queryparser/traversal_proto.go | 8 +- 18 files changed, 560 insertions(+), 90 deletions(-) diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql index 435adb925..157d311ce 100644 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql @@ -32,4 +32,15 @@ CREATE VIEW public.orders_view AS pg_try_advisory_lock((hashtext((orders.customer_name || orders.product_name)))::bigint) AS lock_acquired, orders.ctid AS row_ctid, orders.xmin AS transaction_id - FROM public.orders; \ No newline at end of file + FROM public.orders; + +CREATE VIEW public.my_films_view AS +SELECT jt.* FROM + my_films, + JSON_TABLE ( js, '$.favorites[*]' + COLUMNS ( + id FOR ORDINALITY, + kind text PATH '$.kind', + NESTED PATH '$.films[*]' COLUMNS ( + title text FORMAT JSON PATH '$.title' OMIT QUOTES, + director text PATH '$.director' KEEP QUOTES))) AS jt; \ No newline at end of file diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index 35cc21ec2..28e11bd8c 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -20,6 +20,36 @@ "GH": "https://github.com/yugabyte/yb-voyager/issues/1542", "MinimumVersionsFixedIn": null }, + { + "IssueType": "unsupported_features", + "ObjectType": "VIEW", + "ObjectName": "public.my_films_view", + "Reason": "Json Query Functions", + "SqlStatement": "CREATE VIEW public.my_films_view AS\nSELECT jt.* FROM\n my_films,\n JSON_TABLE ( js, '$.favorites[*]'\n COLUMNS (\n id FOR ORDINALITY,\n kind text PATH '$.kind',\n NESTED PATH '$.films[*]' COLUMNS (\n title text FORMAT JSON PATH '$.title' OMIT QUOTES,\n director text PATH '$.director' KEEP QUOTES))) AS jt;", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "VIEW", + "ObjectName": "test", + "Reason": "Json Constructor Functions", + "SqlStatement": "CREATE OR REPLACE view test AS (\n select x , JSON_ARRAYAGG(trunc(b, 2) order by t desc) as agg\n FROM test1\n where t = '1DAY' group by x\n );", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "MVIEW", + "ObjectName": "test", + "Reason": "Json Constructor Functions", + "SqlStatement": "CREATE MATERIALIZED VIEW test AS (\n select x , JSON_ARRAYAGG(trunc(b, 2) order by t desc) as agg\n FROM test1\n where t = '1DAY' group by x\n );", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null + }, { "IssueType": "unsupported_features", "ObjectType": "TABLE", diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index 969e87da8..f2d1304f3 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -50,9 +50,9 @@ }, { "ObjectType": "VIEW", - "TotalCount": 5, - "InvalidCount": 5, - "ObjectNames": "v1, v2, test, public.orders_view, view_name" + "TotalCount": 6, + "InvalidCount": 6, + "ObjectNames": "public.my_films_view, v1, v2, test, public.orders_view, view_name" }, { "ObjectType": "TRIGGER", diff --git a/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql b/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql index c4eff19e4..2b42d3334 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql +++ b/migtests/tests/pg/assessment-report-test-uqc/pg_assessment_report_uqc.sql @@ -41,4 +41,8 @@ INSERT INTO hr.departments (department_name, location) VALUES ('Engineering', 'B INSERT INTO hr.departments (department_name, location) VALUES ('Sales', 'Building B'); INSERT INTO public.employees (name, department_id) VALUES ('Alice', 1), ('Bob', 1), ('Charlie', 2); INSERT INTO sales.orders (customer_id, amount) VALUES (101, 500.00), (102, 1200.00); -INSERT INTO analytics.metrics (metric_name, metric_value) VALUES ('ConversionRate', 0.023), ('ChurnRate', 0.05); \ No newline at end of file +INSERT INTO analytics.metrics (metric_name, metric_value) VALUES ('ConversionRate', 0.023), ('ChurnRate', 0.05); + +create view sales.employ_depart_view AS SELECT + any_value(name) AS any_employee + FROM employees; \ No newline at end of file diff --git a/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql b/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql index a0ebe3d97..1840c3624 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql +++ b/migtests/tests/pg/assessment-report-test-uqc/unsupported_query_constructs.sql @@ -20,4 +20,9 @@ FROM public.employees; -- 4) Advisory locks (analytics schema) SELECT metric_name, pg_advisory_lock(metric_id) FROM analytics.metrics -WHERE metric_value > 0.02; \ No newline at end of file +WHERE metric_value > 0.02; + +-- Aggregate functions UQC NOT REPORTING as it need PG16 upgarde in pipeline from PG15 +SELECT + any_value(name) AS any_employee + FROM employees; \ No newline at end of file diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 945c42e9a..12cae929b 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -1023,6 +1023,9 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SYSTEM_COLUMNS_NAME, queryissue.SYSTEM_COLUMNS_NAME, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.LARGE_OBJECT_FUNCTIONS_NAME, queryissue.LARGE_OBJECT_FUNCTIONS_NAME, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REGEX_FUNCTIONS_FEATURE, "", queryissue.REGEX_FUNCTIONS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_QUERY_FUNCTIONS_NAME, "", queryissue.JSON_QUERY_FUNCTION, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_CONSTRUCTOR_FUNCTION_NAME, "", queryissue.JSON_CONSTRUCTOR_FUNCTION, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.AGGREGATION_FUNCTIONS_NAME, "", queryissue.AGGREGATE_FUNCTION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SECURITY_INVOKER_VIEWS_NAME, "", queryissue.SECURITY_INVOKER_VIEWS, schemaAnalysisReport, false, "")) return lo.Filter(unsupportedFeatures, func(f UnsupportedFeature, _ int) bool { diff --git a/yb-voyager/cmd/templates/migration_assessment_report.template b/yb-voyager/cmd/templates/migration_assessment_report.template index 6681f0eae..6fe380e1e 100644 --- a/yb-voyager/cmd/templates/migration_assessment_report.template +++ b/yb-voyager/cmd/templates/migration_assessment_report.template @@ -286,7 +286,7 @@ {{ if $docsLink }} Docs Link {{ else }} - Not Available + N/A {{ end }} @@ -313,6 +313,7 @@ {{ $objectsGroupByObjectType := groupByObjectType .Objects }} {{ $numUniqueObjectNamesOfAllTypes := totalUniqueObjectNamesOfAllTypes $objectsGroupByObjectType }} {{ $docsLink := .DocsLink }} + {{ $supportedVerStr := getSupportedVersionString .MinimumVersionsFixedIn }} {{ .FeatureName }} {{ $isNextRowRequiredForObjectType := false }} {{ range $type, $objectsByType := $objectsGroupByObjectType }} @@ -338,7 +339,16 @@ {{ if not $isNextRowRequiredForObjectType }} - Link + + {{ if $supportedVerStr }} + Supported in Versions: {{ $supportedVerStr }}
+ {{ end }} + {{ if $docsLink }} + Docs Link + {{ else }} + N/A + {{ end }} + {{ end }} {{ $isNextRowRequiredForObjectName = true }} {{ $isNextRowRequiredForObjectType = true }} @@ -359,6 +369,10 @@ {{ $hasMigrationCaveats = true }} {{if .DisplayDDL }}

{{.FeatureName}}

+ {{ $supporterVerStr := getSupportedVersionString .MinimumVersionsFixedIn }} + {{ if $supporterVerStr }} +

Supported in Versions: {{ $supporterVerStr }}

+ {{ end }}

{{.FeatureDescription}}

    @@ -369,6 +383,10 @@
{{else}}

{{.FeatureName}}

+ {{ $supporterVerStr := getSupportedVersionString .MinimumVersionsFixedIn }} + {{ if $supporterVerStr }} +

Supported in Versions: {{ $supporterVerStr }}

+ {{ end }}

{{.FeatureDescription}}

    diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index af2d24a6c..f0f86239a 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -49,9 +49,15 @@ const ( FOREIGN_TABLE = "FOREIGN_TABLE" INHERITANCE = "INHERITANCE" - LARGE_OBJECT_DATATYPE = "LARGE_OBJECT_DATATYPE" - LARGE_OBJECT_FUNCTIONS = "LARGE_OBJECT_FUNCTIONS" - LARGE_OBJECT_FUNCTIONS_NAME = "Large Object Functions" + AGGREGATE_FUNCTION = "AGGREGATE_FUNCTION" + AGGREGATION_FUNCTIONS_NAME = "Aggregate Functions" + JSON_CONSTRUCTOR_FUNCTION = "JSON_CONSTRUCTOR_FUNCTION" + JSON_CONSTRUCTOR_FUNCTION_NAME = "Json Constructor Functions" + JSON_QUERY_FUNCTION = "JSON_QUERY_FUNCTION" + JSON_QUERY_FUNCTIONS_NAME = "Json Query Functions" + LARGE_OBJECT_DATATYPE = "LARGE_OBJECT_DATATYPE" + LARGE_OBJECT_FUNCTIONS = "LARGE_OBJECT_FUNCTIONS" + LARGE_OBJECT_FUNCTIONS_NAME = "Large Object Functions" SECURITY_INVOKER_VIEWS = "SECURITY_INVOKER_VIEWS" SECURITY_INVOKER_VIEWS_NAME = "Security Invoker Views" @@ -69,6 +75,7 @@ const ( // Object types const ( CONSTRAINT_NAME = "ConstraintName" + FUNCTION_NAMES = "FunctionNames" TABLE_OBJECT_TYPE = "TABLE" FOREIGN_TABLE_OBJECT_TYPE = "FOREIGN TABLE" FUNCTION_OBJECT_TYPE = "FUNCTION" diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index bb9490786..8a57c581d 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -35,6 +35,7 @@ type FuncCallDetector struct { advisoryLocksFuncsDetected mapset.Set[string] xmlFuncsDetected mapset.Set[string] + aggFuncsDetected mapset.Set[string] regexFuncsDetected mapset.Set[string] loFuncsDetected mapset.Set[string] } @@ -44,6 +45,7 @@ func NewFuncCallDetector(query string) *FuncCallDetector { query: query, advisoryLocksFuncsDetected: mapset.NewThreadUnsafeSet[string](), xmlFuncsDetected: mapset.NewThreadUnsafeSet[string](), + aggFuncsDetected: mapset.NewThreadUnsafeSet[string](), regexFuncsDetected: mapset.NewThreadUnsafeSet[string](), loFuncsDetected: mapset.NewThreadUnsafeSet[string](), } @@ -68,6 +70,9 @@ func (d *FuncCallDetector) Detect(msg protoreflect.Message) error { d.regexFuncsDetected.Add(funcName) } + if unsupportedAggFunctions.ContainsOne(funcName) { + d.aggFuncsDetected.Add(funcName) + } if unsupportedLargeObjectFunctions.ContainsOne(funcName) { d.loFuncsDetected.Add(funcName) } @@ -83,11 +88,14 @@ func (d *FuncCallDetector) GetIssues() []QueryIssue { if d.xmlFuncsDetected.Cardinality() > 0 { issues = append(issues, NewXmlFunctionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) } + if d.aggFuncsDetected.Cardinality() > 0 { + issues = append(issues, NewAggregationFunctionIssue(DML_QUERY_OBJECT_TYPE, "", d.query, d.aggFuncsDetected.ToSlice())) + } if d.regexFuncsDetected.Cardinality() > 0 { issues = append(issues, NewRegexFunctionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) } if d.loFuncsDetected.Cardinality() > 0 { - issues = append(issues, NewLOFuntionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) + issues = append(issues, NewLOFuntionsIssue(DML_QUERY_OBJECT_TYPE, "", d.query, d.loFuncsDetected.ToSlice())) } return issues } @@ -195,3 +203,105 @@ func (d *RangeTableFuncDetector) GetIssues() []QueryIssue { } return issues } + +type JsonConstructorFuncDetector struct { + query string + unsupportedJsonConstructorFunctionsDetected mapset.Set[string] +} + +func NewJsonConstructorFuncDetector(query string) *JsonConstructorFuncDetector { + return &JsonConstructorFuncDetector{ + query: query, + unsupportedJsonConstructorFunctionsDetected: mapset.NewThreadUnsafeSet[string](), + } +} + +func (j *JsonConstructorFuncDetector) Detect(msg protoreflect.Message) error { + switch queryparser.GetMsgFullName(msg) { + case queryparser.PG_QUERY_JSON_ARRAY_AGG_NODE: + j.unsupportedJsonConstructorFunctionsDetected.Add(JSON_ARRAYAGG) + case queryparser.PG_QUERY_JSON_ARRAY_CONSTRUCTOR_AGG_NODE: + j.unsupportedJsonConstructorFunctionsDetected.Add(JSON_ARRAY) + case queryparser.PG_QUERY_JSON_OBJECT_AGG_NODE: + j.unsupportedJsonConstructorFunctionsDetected.Add(JSON_OBJECTAGG) + case queryparser.PG_QUERY_JSON_OBJECT_CONSTRUCTOR_NODE: + j.unsupportedJsonConstructorFunctionsDetected.Add(JSON_OBJECT) + } + return nil +} + +func (d *JsonConstructorFuncDetector) GetIssues() []QueryIssue { + var issues []QueryIssue + if d.unsupportedJsonConstructorFunctionsDetected.Cardinality() > 0 { + issues = append(issues, NewJsonConstructorFunctionIssue(DML_QUERY_OBJECT_TYPE, "", d.query, d.unsupportedJsonConstructorFunctionsDetected.ToSlice())) + } + return issues +} + +type JsonQueryFunctionDetector struct { + query string + unsupportedJsonQueryFunctionsDetected mapset.Set[string] +} + +func NewJsonQueryFunctionDetector(query string) *JsonQueryFunctionDetector { + return &JsonQueryFunctionDetector{ + query: query, + unsupportedJsonQueryFunctionsDetected: mapset.NewThreadUnsafeSet[string](), + } +} + +func (j *JsonQueryFunctionDetector) Detect(msg protoreflect.Message) error { + if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_JSON_TABLE_NODE { + /* + SELECT * FROM json_table( + '[{"a":10,"b":20},{"a":30,"b":40}]'::jsonb, + '$[*]' + COLUMNS ( + column_a int4 path '$.a', + column_b int4 path '$.b' + ) + ); + stmts:{stmt:{select_stmt:{target_list:{res_target:{val:{column_ref:{fields:{a_star:{}} location:530}} location:530}} + from_clause:{json_table:{context_item:{raw_expr:{type_cast:{arg:{a_const:{sval:{sval:"[{\"a\":10,\"b\":20},{\"a\":30,\"b\":40}]"} + location:553}} type_name:{names:{string:{sval:"jsonb"}} ..... name_location:-1 location:601} + columns:{json_table_column:{coltype:JTC_REGULAR name:"column_a" type_name:{names:{string:{sval:"int4"}} typemod:-1 location:639} + pathspec:{string:{a_const:{sval:{sval:"$.a"} location:649}} name_location:-1 location:649} ... + */ + j.unsupportedJsonQueryFunctionsDetected.Add(JSON_TABLE) + return nil + } + if queryparser.GetMsgFullName(msg) != queryparser.PG_QUERY_JSON_FUNC_EXPR_NODE { + return nil + } + /* + JsonExprOp - + enumeration of SQL/JSON query function types + typedef enum JsonExprOp + { + 1. JSON_EXISTS_OP, JSON_EXISTS() + 2. JSON_QUERY_OP, JSON_QUERY() + 3. JSON_VALUE_OP, JSON_VALUE() + 4. JSON_TABLE_OP, JSON_TABLE() + } JsonExprOp; + */ + jsonExprFuncOpNum := queryparser.GetEnumNumField(msg, "op") + switch jsonExprFuncOpNum { + case 1: + j.unsupportedJsonQueryFunctionsDetected.Add(JSON_EXISTS) + case 2: + j.unsupportedJsonQueryFunctionsDetected.Add(JSON_QUERY) + case 3: + j.unsupportedJsonQueryFunctionsDetected.Add(JSON_VALUE) + case 4: + j.unsupportedJsonQueryFunctionsDetected.Add(JSON_TABLE) + } + return nil +} + +func (d *JsonQueryFunctionDetector) GetIssues() []QueryIssue { + var issues []QueryIssue + if d.unsupportedJsonQueryFunctionsDetected.Cardinality() > 0 { + issues = append(issues, NewJsonQueryFunctionIssue(DML_QUERY_OBJECT_TYPE, "", d.query, d.unsupportedJsonQueryFunctionsDetected.ToSlice())) + } + return issues +} diff --git a/yb-voyager/src/query/queryissue/detectors_ddl.go b/yb-voyager/src/query/queryissue/detectors_ddl.go index 532ee2dc2..82ea56d04 100644 --- a/yb-voyager/src/query/queryissue/detectors_ddl.go +++ b/yb-voyager/src/query/queryissue/detectors_ddl.go @@ -551,6 +551,7 @@ func (tid *TriggerIssueDetector) DetectIssues(obj queryparser.DDLObject) ([]Quer obj.GetObjectType(), trigger.GetObjectName(), "", + []string{trigger.FuncName}, )) } diff --git a/yb-voyager/src/query/queryissue/detectors_test.go b/yb-voyager/src/query/queryissue/detectors_test.go index 48b8d674c..266f053ba 100644 --- a/yb-voyager/src/query/queryissue/detectors_test.go +++ b/yb-voyager/src/query/queryissue/detectors_test.go @@ -27,6 +27,26 @@ import ( "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryparser" ) +func getDetectorIssues(t *testing.T, detector UnsupportedConstructDetector, sql string) []QueryIssue { + parseResult, err := queryparser.Parse(sql) + assert.NoError(t, err, "Failed to parse SQL: %s", sql) + + visited := make(map[protoreflect.Message]bool) + + processor := func(msg protoreflect.Message) error { + err := detector.Detect(msg) + if err != nil { + return err + } + return nil + } + + parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) + err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) + assert.NoError(t, err) + return detector.GetIssues() +} + func TestFuncCallDetector(t *testing.T) { advisoryLockSqls := []string{ `SELECT pg_advisory_lock(100), COUNT(*) FROM cars;`, @@ -76,6 +96,13 @@ func TestFuncCallDetector(t *testing.T) { `SELECT pg_advisory_unlock_all();`, } + anyValAggSqls := []string{ + `SELECT + department, + any_value(employee_name) AS any_employee + FROM employees + GROUP BY department;`, + } loFunctionSqls := []string{ `UPDATE documents SET content_oid = lo_import('/path/to/new/file.pdf') @@ -90,39 +117,23 @@ WHERE title = 'Design Document';`, `SELECT lo_unlink((SELECT content_oid FROM documents WHERE title = 'Sample Document'));`, `create table test_lo_default (id int, raster lo DEFAULT lo_import('3242'));`, } - - detectConstructs := func(sql string) []QueryIssue { - detector := NewFuncCallDetector(sql) - parseResult, err := queryparser.Parse(sql) - assert.NoError(t, err, "Failed to parse SQL: %s", sql) - - visited := make(map[protoreflect.Message]bool) - - processor := func(msg protoreflect.Message) error { - err := detector.Detect(msg) - if err != nil { - return err - } - return nil - } - - parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) - err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) - assert.NoError(t, err) - return detector.GetIssues() - } - for _, sql := range advisoryLockSqls { - issues := detectConstructs(sql) - assert.Equal(t, len(issues), 1) - assert.Equal(t, issues[0].Type, ADVISORY_LOCKS, "Advisory Locks not detected in SQL: %s", sql) + + issues := getDetectorIssues(t, NewFuncCallDetector(sql), sql) + assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql) + assert.Equal(t, ADVISORY_LOCKS, issues[0].Type, "Expected Advisory Locks issue for SQL: %s", sql) } for _, sql := range loFunctionSqls { - issues := detectConstructs(sql) + issues := getDetectorIssues(t, NewFuncCallDetector(sql), sql) assert.Equal(t, len(issues), 1) assert.Equal(t, issues[0].Type, LARGE_OBJECT_FUNCTIONS, "Large Objects not detected in SQL: %s", sql) + } + for _, sql := range anyValAggSqls { + issues := getDetectorIssues(t, NewFuncCallDetector(sql), sql) + assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql) + assert.Equal(t, AGGREGATE_FUNCTION, issues[0].Type, "Expected Advisory Locks issue for SQL: %s", sql) } } @@ -168,25 +179,7 @@ func TestColumnRefDetector(t *testing.T) { } for _, sql := range systemColumnSqls { - detector := NewColumnRefDetector(sql) - parseResult, err := queryparser.Parse(sql) - assert.NoError(t, err, "Failed to parse SQL: %s", sql) - - visited := make(map[protoreflect.Message]bool) - - processor := func(msg protoreflect.Message) error { - err := detector.Detect(msg) - if err != nil { - return err - } - return nil - } - - parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) - err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) - assert.NoError(t, err) - - issues := detector.GetIssues() + issues := getDetectorIssues(t, NewColumnRefDetector(sql), sql) assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql) assert.Equal(t, SYSTEM_COLUMNS, issues[0].Type, "Expected System Columns issue for SQL: %s", sql) @@ -354,25 +347,7 @@ func TestRangeTableFuncDetector(t *testing.T) { } for _, sql := range xmlTableSqls { - detector := NewRangeTableFuncDetector(sql) - parseResult, err := queryparser.Parse(sql) - assert.NoError(t, err, "Failed to parse SQL: %s", sql) - - visited := make(map[protoreflect.Message]bool) - - processor := func(msg protoreflect.Message) error { - err := detector.Detect(msg) - if err != nil { - return err - } - return nil - } - - parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) - err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) - assert.NoError(t, err) - - issues := detector.GetIssues() + issues := getDetectorIssues(t, NewRangeTableFuncDetector(sql), sql) assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql) assert.Equal(t, XML_FUNCTIONS, issues[0].Type, "Expected XML Functions issue for SQL: %s", sql) @@ -713,3 +688,23 @@ func TestCombinationOfDetectors1WithObjectCollector(t *testing.T) { "Schema list mismatch for sql [%s]. Expected: %v(len=%d), Actual: %v(len=%d)", tc.Sql, tc.ExpectedSchemas, len(tc.ExpectedSchemas), collectedSchemas, len(collectedSchemas)) } } + +func TestJsonConstructorDetector(t *testing.T) { + sql := `SELECT JSON_ARRAY('PostgreSQL', 12, TRUE, NULL) AS json_array;` + + issues := getDetectorIssues(t, NewJsonConstructorFuncDetector(sql), sql) + assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql) + assert.Equal(t, JSON_CONSTRUCTOR_FUNCTION, issues[0].Type, "Expected Advisory Locks issue for SQL: %s", sql) + +} + +func TestJsonQueryFunctionDetector(t *testing.T) { + sql := `SELECT id, JSON_VALUE(details, '$.title') AS title +FROM books +WHERE JSON_EXISTS(details, '$.price ? (@ > $price)' PASSING 30 AS price);` + + issues := getDetectorIssues(t, NewJsonQueryFunctionDetector(sql), sql) + assert.Equal(t, 1, len(issues), "Expected 1 issue for SQL: %s", sql) + assert.Equal(t, JSON_QUERY_FUNCTION, issues[0].Type, "Expected Advisory Locks issue for SQL: %s", sql) + +} diff --git a/yb-voyager/src/query/queryissue/helpers.go b/yb-voyager/src/query/queryissue/helpers.go index 617b2fb70..d24a14ed5 100644 --- a/yb-voyager/src/query/queryissue/helpers.go +++ b/yb-voyager/src/query/queryissue/helpers.go @@ -102,6 +102,24 @@ var UnsupportedIndexDatatypes = []string{ // array as well but no need to add it in the list as fetching this type is a different way TODO: handle better with specific types } +var unsupportedAggFunctions = mapset.NewThreadUnsafeSet([]string{ + //agg function added in PG16 - https://www.postgresql.org/docs/16/functions-aggregate.html#id-1.5.8.27.5.2.4.1.1.1.1 + "any_value", +}...) + +const ( + // // json functions, refer - https://www.postgresql.org/about/featurematrix/detail/395/ + JSON_OBJECTAGG = "JSON_OBJECTAGG" + JSON_ARRAY = "JSON_ARRAY" + JSON_ARRAYAGG = "JSON_ARRAYAGG" + JSON_OBJECT = "JSON_OBJECT" + //json query functions supported in PG 17, refer - https://www.postgresql.org/docs/17/functions-json.html#FUNCTIONS-SQLJSON-QUERYING + JSON_EXISTS = "JSON_EXISTS" + JSON_QUERY = "JSON_QUERY" + JSON_VALUE = "JSON_VALUE" + JSON_TABLE = "JSON_TABLE" +) + var unsupportedLargeObjectFunctions = mapset.NewThreadUnsafeSet([]string{ //refer - https://www.postgresql.org/docs/current/lo-interfaces.html#LO-CREATE diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index 33db98be0..43dc8ab07 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -16,7 +16,11 @@ limitations under the License. package queryissue -import "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" +import ( + "sort" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" +) var advisoryLocksIssue = issue.Issue{ Type: ADVISORY_LOCKS, @@ -70,6 +74,57 @@ func NewRegexFunctionsIssue(objectType string, objectName string, sqlStatement s return newQueryIssue(regexFunctionsIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } +var anyValueAggFunctionIssue = issue.Issue{ + Type: AGGREGATE_FUNCTION, + TypeName: AGGREGATION_FUNCTIONS_NAME, + TypeDescription: "Postgresql 17 features not supported yet in YugabyteDB", + Suggestion: "", + GH: "", + DocsLink: "", +} + +func NewAggregationFunctionIssue(objectType string, objectName string, sqlStatement string, funcNames []string) QueryIssue { + sort.Strings(funcNames) + details := map[string]interface{}{ + FUNCTION_NAMES: funcNames, //TODO USE it later when we start putting these in reports + } + return newQueryIssue(anyValueAggFunctionIssue, objectType, objectName, sqlStatement, details) +} + +var jsonConstructorFunctionsIssue = issue.Issue{ + Type: JSON_CONSTRUCTOR_FUNCTION, + TypeName: JSON_CONSTRUCTOR_FUNCTION_NAME, + TypeDescription: "Postgresql 17 features not supported yet in YugabyteDB", + Suggestion: "", + GH: "", + DocsLink: "", +} + +func NewJsonConstructorFunctionIssue(objectType string, objectName string, sqlStatement string, funcNames []string) QueryIssue { + sort.Strings(funcNames) + details := map[string]interface{}{ + FUNCTION_NAMES: funcNames, //TODO USE it later when we start putting these in reports + } + return newQueryIssue(jsonConstructorFunctionsIssue, objectType, objectName, sqlStatement, details) +} + +var jsonQueryFunctionIssue = issue.Issue{ + Type: JSON_QUERY_FUNCTION, + TypeName: JSON_QUERY_FUNCTIONS_NAME, + TypeDescription: "Postgresql 17 features not supported yet in YugabyteDB", + Suggestion: "", + GH: "", + DocsLink: "", +} + +func NewJsonQueryFunctionIssue(objectType string, objectName string, sqlStatement string, funcNames []string) QueryIssue { + sort.Strings(funcNames) + details := map[string]interface{}{ + FUNCTION_NAMES: funcNames, //TODO USE it later when we start putting these in reports + } + return newQueryIssue(jsonQueryFunctionIssue, objectType, objectName, sqlStatement, details) +} + var loFunctionsIssue = issue.Issue{ Type: LARGE_OBJECT_FUNCTIONS, TypeName: LARGE_OBJECT_FUNCTIONS_NAME, @@ -79,6 +134,10 @@ var loFunctionsIssue = issue.Issue{ DocsLink: "", //TODO } -func NewLOFuntionsIssue(objectType string, objectName string, sqlStatement string) QueryIssue { - return newQueryIssue(loFunctionsIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +func NewLOFuntionsIssue(objectType string, objectName string, sqlStatement string, funcNames []string) QueryIssue { + sort.Strings(funcNames) + details := map[string]interface{}{ + FUNCTION_NAMES: funcNames, //TODO USE it later when we start putting these in reports + } + return newQueryIssue(loFunctionsIssue, objectType, objectName, sqlStatement, details) } diff --git a/yb-voyager/src/query/queryissue/issues_dml_test.go b/yb-voyager/src/query/queryissue/issues_dml_test.go index af3d8afcc..5b0cd0151 100644 --- a/yb-voyager/src/query/queryissue/issues_dml_test.go +++ b/yb-voyager/src/query/queryissue/issues_dml_test.go @@ -46,7 +46,6 @@ func testRegexFunctionsIssue(t *testing.T) { assert.NoError(t, err) defer conn.Close(context.Background()) - stmts := []string{ `SELECT regexp_count('This is an example. Another example. Example is a common word.', 'example')`, `SELECT regexp_instr('This is an example. Another example. Example is a common word.', 'example')`, @@ -59,6 +58,62 @@ func testRegexFunctionsIssue(t *testing.T) { } } +func testJsonConstructorFunctions(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + sqls := map[string]string{ + `select json_object('code' VALUE 'P123', 'title': 'Jaws');`: `syntax error at or near "VALUE"`, + `select JSON_ARRAYAGG('[1, "2", null]');`: `does not exist`, + `SELECT json_objectagg(k VALUE v) AS json_result + FROM (VALUES ('a', 1), ('b', 2), ('c', 3)) AS t(k, v);`: `syntax error at or near "VALUE"`, + `SELECT JSON_ARRAY('PostgreSQL', 12, TRUE, NULL) AS json_array;`: `does not exist`, + } + for sql, expectedErr := range sqls { + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, sql) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, expectedErr, jsonConstructorFunctionsIssue) + } +} + +func testJsonQueryFunctions(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + sqls := []string{ + `SELECT id, JSON_QUERY(details, '$.author') AS author +FROM books;`, + `SELECT + id, + JSON_VALUE(details, '$.title') AS title, + JSON_VALUE(details, '$.price')::NUMERIC AS price +FROM books;`, + `SELECT id, details +FROM books +WHERE JSON_EXISTS(details, '$.author');`, + } + for _, sql := range sqls { + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, sql) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, `does not exist`, jsonConstructorFunctionsIssue) + } + + jsonTableSQL := `SELECT * FROM json_table( + '[{"a":10,"b":20},{"a":30,"b":40}]'::jsonb, + '$[*]' + COLUMNS ( + column_a int4 path '$.a', + column_b int4 path '$.b' + ) + );` + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, jsonTableSQL) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, `syntax error at or near "COLUMNS"`, jsonConstructorFunctionsIssue) +} + func TestDMLIssuesInYBVersion(t *testing.T) { var err error ybVersion := os.Getenv("YB_VERSION") @@ -90,4 +145,10 @@ func TestDMLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "regex functions", ybVersion), testRegexFunctionsIssue) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "json constructor functions", ybVersion), testJsonConstructorFunctions) + assert.True(t, success) + + success = t.Run(fmt.Sprintf("%s-%s", "json query functions", ybVersion), testJsonQueryFunctions) + assert.True(t, success) + } diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector.go b/yb-voyager/src/query/queryissue/parser_issue_detector.go index ffa1d9860..8b5f028b9 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector.go @@ -375,6 +375,8 @@ func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) NewColumnRefDetector(query), NewXmlExprDetector(query), NewRangeTableFuncDetector(query), + NewJsonConstructorFuncDetector(query), + NewJsonQueryFunctionDetector(query), } processor := func(msg protoreflect.Message) error { diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index 066d738fb..1450f4edd 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -278,7 +278,7 @@ func TestDDLIssues(t *testing.T) { }, stmt19: []QueryIssue{ NewLODatatypeIssue("TABLE", "test_lo_default", stmt19, "raster"), - NewLOFuntionsIssue("TABLE", "test_lo_default", stmt19), + NewLOFuntionsIssue("TABLE", "test_lo_default", stmt19, []string{"lo_import"}), }, stmt20: []QueryIssue{ NewSecurityInvokerViewIssue("VIEW", "public.view_explicit_security_invoker", stmt20), @@ -428,25 +428,25 @@ $$ LANGUAGE plpgsql; expectedSQLsWithIssues := map[string][]QueryIssue{ sqls[0]: []QueryIssue{ - NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_unlink(loid);"), + NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_unlink(loid);", []string{"lo_unlink"}), }, sqls[1]: []QueryIssue{ - NewLOFuntionsIssue("DML_QUERY", "", "INSERT INTO documents (title, content_oid) VALUES (doc_title, lo_import(file_path));"), + NewLOFuntionsIssue("DML_QUERY", "", "INSERT INTO documents (title, content_oid) VALUES (doc_title, lo_import(file_path));", []string{"lo_import"}), }, sqls[2]: []QueryIssue{ - NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_export(loid, file_path);"), + NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_export(loid, file_path);", []string{"lo_export"}), }, sqls[3]: []QueryIssue{ - NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_close(fd);"), + NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_close(fd);", []string{"lo_close"}), }, sqls[4]: []QueryIssue{ - NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_put(fd, convert_to(new_data, 'UTF8'));"), - NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_close(fd);"), + NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_put(fd, convert_to(new_data, 'UTF8'));", []string{"lo_put"}), + NewLOFuntionsIssue("DML_QUERY", "", "SELECT lo_close(fd);", []string{"lo_close"}), NewLODatatypeIssue("TABLE", "test_large_objects", "CREATE TABLE IF NOT EXISTS test_large_objects(id INT, raster lo DEFAULT lo_import(3242));", "raster"), - NewLOFuntionsIssue("TABLE", "test_large_objects", "CREATE TABLE IF NOT EXISTS test_large_objects(id INT, raster lo DEFAULT lo_import(3242));"), + NewLOFuntionsIssue("TABLE", "test_large_objects", "CREATE TABLE IF NOT EXISTS test_large_objects(id INT, raster lo DEFAULT lo_import(3242));", []string{"lo_import"}), }, sqls[5]: []QueryIssue{ - NewLOFuntionsIssue("TRIGGER", "t_raster ON image", sqls[5]), + NewLOFuntionsIssue("TRIGGER", "t_raster ON image", sqls[5], []string{"lo_manage"}), }, } expectedSQLsWithIssues[sqls[0]] = modifiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[0]], "FUNCTION", "manage_large_object") @@ -492,6 +492,134 @@ func TestSingleXMLIssueIsDetected(t *testing.T) { assert.Equal(t, 1, len(issues)) } +func TestJsonUnsupportedFeatures(t *testing.T) { + sqls := []string{ + `SELECT department, JSON_ARRAYAGG(name) AS employees_json + FROM employees + GROUP BY department;`, + `INSERT INTO movies (details) +VALUES ( + JSON_OBJECT('title' VALUE 'Dune', 'director' VALUE 'Denis Villeneuve', 'year' VALUE 2021) +);`, + `SELECT json_objectagg(k VALUE v) AS json_result + FROM (VALUES ('a', 1), ('b', 2), ('c', 3)) AS t(k, v);`, + `SELECT JSON_OBJECT( + 'movie' VALUE JSON_OBJECT('code' VALUE 'P123', 'title' VALUE 'Jaws'), + 'director' VALUE 'Steven Spielberg' +) AS nested_json_object;`, + `select JSON_ARRAYAGG('[1, "2", null]');`, + `SELECT JSON_OBJECT( + 'code' VALUE 'P123', + 'title' VALUE 'Jaws', + 'price' VALUE 19.99, + 'available' VALUE TRUE +) AS json_obj;`, + `SELECT id, JSON_QUERY(details, '$.author') AS author +FROM books;`, + `SELECT jt.* FROM + my_films, + JSON_TABLE (js, '$.favorites[*]' COLUMNS ( + id FOR ORDINALITY, + kind text PATH '$.kind', + title text PATH '$.films[*].title' WITH WRAPPER, + director text PATH '$.films[*].director' WITH WRAPPER)) AS jt;`, + `SELECT jt.* FROM + my_films, + JSON_TABLE (js, $1 COLUMNS ( + id FOR ORDINALITY, + kind text PATH '$.kind', + title text PATH '$.films[*].title' WITH WRAPPER, + director text PATH '$.films[*].director' WITH WRAPPER)) AS jt;`, + `SELECT id, details +FROM books +WHERE JSON_EXISTS(details, '$.author');`, + `SELECT id, JSON_QUERY(details, '$.author') AS author +FROM books;`, + `SELECT + id, + JSON_VALUE(details, '$.title') AS title, + JSON_VALUE(details, '$.price')::NUMERIC AS price +FROM books;`, + `SELECT id, JSON_VALUE(details, '$.title') AS title +FROM books +WHERE JSON_EXISTS(details, '$.price ? (@ > $price)' PASSING 30 AS price);`, +`CREATE MATERIALIZED VIEW public.test_jsonb_view AS +SELECT + id, + data->>'name' AS name, + JSON_VALUE(data, '$.age' RETURNING INTEGER) AS age, + JSON_EXISTS(data, '$.skills[*] ? (@ == "JSON")') AS knows_json, + jt.skill +FROM public.test_jsonb, +JSON_TABLE(data, '$.skills[*]' + COLUMNS ( + skill TEXT PATH '$' + ) +) AS jt;`, + `SELECT JSON_ARRAY($1, 12, TRUE, $2) AS json_array;`, + } + sqlsWithExpectedIssues := map[string][]QueryIssue{ + sqls[0]: []QueryIssue{ + NewJsonConstructorFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[0], []string{JSON_ARRAYAGG}), + }, + sqls[1]: []QueryIssue{ + NewJsonConstructorFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[1], []string{JSON_OBJECT}), + }, + sqls[2]: []QueryIssue{ + NewJsonConstructorFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[2], []string{JSON_OBJECTAGG}), + }, + sqls[3]: []QueryIssue{ + NewJsonConstructorFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[3], []string{JSON_OBJECT}), + }, + sqls[4]: []QueryIssue{ + NewJsonConstructorFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[4], []string{JSON_ARRAYAGG}), + }, + sqls[5]: []QueryIssue{ + NewJsonConstructorFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[5], []string{JSON_OBJECT}), + }, + sqls[6]: []QueryIssue{ + NewJsonQueryFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[6], []string{JSON_QUERY}), + }, + sqls[7]: []QueryIssue{ + NewJsonQueryFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[7], []string{JSON_TABLE}), + }, + // sqls[8]: []QueryIssue{ + // NewJsonQueryFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[8]), + //NOT REPORTED YET because of PARSER failing if JSON_TABLE has a parameterized values $1, $2 ... + //https://github.com/pganalyze/pg_query_go/issues/127 + // }, + sqls[9]: []QueryIssue{ + NewJsonQueryFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[9], []string{JSON_EXISTS}), + }, + sqls[10]: []QueryIssue{ + NewJsonQueryFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[10], []string{JSON_QUERY}), + }, + sqls[11]: []QueryIssue{ + NewJsonQueryFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[11], []string{JSON_VALUE}), + }, + sqls[12]: []QueryIssue{ + NewJsonQueryFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[12], []string{JSON_VALUE, JSON_EXISTS}), + }, + sqls[13]: []QueryIssue{ + NewJsonQueryFunctionIssue("MVIEW", "public.test_jsonb_view", sqls[13], []string{JSON_VALUE, JSON_EXISTS, JSON_TABLE}), + }, + sqls[14]: []QueryIssue{ + NewJsonConstructorFunctionIssue(DML_QUERY_OBJECT_TYPE, "", sqls[14], []string{JSON_ARRAY}), + }, + } + parserIssueDetector := NewParserIssueDetector() + for stmt, expectedIssues := range sqlsWithExpectedIssues { + issues, err := parserIssueDetector.GetAllIssues(stmt, ybversion.LatestStable) + assert.NoError(t, err, "Error detecting issues for statement: %s", stmt) + assert.Equal(t, len(expectedIssues), len(issues), "Mismatch in issue count for statement: %s", stmt) + for _, expectedIssue := range expectedIssues { + found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { + return cmp.Equal(expectedIssue, queryIssue) + }) + assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) + } + } +} func TestRegexFunctionsIssue(t *testing.T) { dmlStmts := []string{ `SELECT regexp_count('This is an example. Another example. Example is a common word.', 'example')`, diff --git a/yb-voyager/src/query/queryparser/helpers_protomsg.go b/yb-voyager/src/query/queryparser/helpers_protomsg.go index c79aebc8c..e0de00604 100644 --- a/yb-voyager/src/query/queryparser/helpers_protomsg.go +++ b/yb-voyager/src/query/queryparser/helpers_protomsg.go @@ -34,6 +34,7 @@ const ( func GetProtoMessageFromParseTree(parseTree *pg_query.ParseResult) protoreflect.Message { return parseTree.Stmts[0].Stmt.ProtoReflect() + } func GetMsgFullName(msg protoreflect.Message) string { @@ -374,6 +375,17 @@ func GetListField(msg protoreflect.Message, fieldName string) protoreflect.List return nil } +// GetEnumNumField retrieves a enum field from a message +// FieldDescriptor{Syntax: proto3, FullName: pg_query.JsonFuncExpr.op, Number: 1, Cardinality: optional, Kind: enum, HasJSONName: true, JSONName: "op", Enum: pg_query.JsonExprOp} +//val:{json_func_expr:{op:JSON_QUERY_OP context_item:{raw_expr:{column_ref:{fields:{string:{sval:"details"}} location:2626}} format:{format_type:JS_FORMAT_DEFAULT encoding:JS_ENC_DEFAULT +func GetEnumNumField(msg protoreflect.Message, fieldName string) protoreflect.EnumNumber { + field := msg.Descriptor().Fields().ByName(protoreflect.Name(fieldName)) + if field != nil && msg.Has(field) { + return msg.Get(field).Enum() + } + return 0 +} + // GetSchemaAndObjectName extracts the schema and object name from a list. func GetSchemaAndObjectName(nameList protoreflect.List) (string, string) { var schemaName, objectName string diff --git a/yb-voyager/src/query/queryparser/traversal_proto.go b/yb-voyager/src/query/queryparser/traversal_proto.go index d54aefe0e..c988ee48e 100644 --- a/yb-voyager/src/query/queryparser/traversal_proto.go +++ b/yb-voyager/src/query/queryparser/traversal_proto.go @@ -42,7 +42,13 @@ const ( PG_QUERY_UPDATESTMT_NODE = "pg_query.UpdateStmt" PG_QUERY_DELETESTMT_NODE = "pg_query.DeleteStmt" - PG_QUERY_VIEWSTMT_NODE = "pg_query.ViewStmt" + PG_QUERY_JSON_OBJECT_AGG_NODE = "pg_query.JsonObjectAgg" + PG_QUERY_JSON_ARRAY_AGG_NODE = "pg_query.JsonArrayAgg" + PG_QUERY_JSON_ARRAY_CONSTRUCTOR_AGG_NODE = "pg_query.JsonArrayConstructor" + PG_QUERY_JSON_FUNC_EXPR_NODE = "pg_query.JsonFuncExpr" + PG_QUERY_JSON_OBJECT_CONSTRUCTOR_NODE = "pg_query.JsonObjectConstructor" + PG_QUERY_JSON_TABLE_NODE = "pg_query.JsonTable" + PG_QUERY_VIEWSTMT_NODE = "pg_query.ViewStmt" ) // function type for processing nodes during traversal From 43aaeeb74d26290fa154995c694a5b25acfed589 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Tue, 24 Dec 2024 19:50:56 +0530 Subject: [PATCH 09/17] Introduce TestContainer interface to streamline test setup of supported databases (#2053) - Any package in voyager code, for eg srcdb, tgtdb, connpool, yugabyted etc.. can spin up the required source db(oracle, pg, mysql) with a specific version to write unit tests. - Every call to TestContainer.Start() check if a required database(dbtype + version) is already run then reuse it, otherwise start a new one - In every package add a TestMain() function, to setup the environment needed specifically for that test. - Added a new Github actions workflow to run the integration tests separately(required some oracle instance client libraries). - Moving forward it will be good to have separation between go tests as unit or integration * Bug fix: GetNonPKTables() for PG and YB returned non table objects like sequences and pk constraint objects - fixed to return only table names --- .github/workflows/go.yml | 4 +- .github/workflows/integration-tests.yml | 53 ++++++ .github/workflows/issue-tests.yml | 3 +- yb-voyager/cmd/common_test.go | 2 +- yb-voyager/cmd/exportDataStatus_test.go | 2 +- yb-voyager/go.mod | 11 +- yb-voyager/go.sum | 15 +- yb-voyager/src/callhome/diagnostics_test.go | 2 +- yb-voyager/src/cp/yugabyted/yugabyted_test.go | 22 ++- yb-voyager/src/datafile/descriptor_test.go | 2 +- yb-voyager/src/dbzm/status_test.go | 2 +- yb-voyager/src/metadb/metadataDB_test.go | 2 +- .../src/migassessment/assessmentDB_test.go | 2 +- yb-voyager/src/namereg/namereg_test.go | 2 +- yb-voyager/src/srcdb/main_test.go | 176 ++++++++++++++++++ yb-voyager/src/srcdb/mysql_test.go | 89 +++++++++ yb-voyager/src/srcdb/oracle_test.go | 80 ++++++++ yb-voyager/src/srcdb/postgres.go | 4 +- yb-voyager/src/srcdb/postgres_test.go | 136 ++++++++++++++ yb-voyager/src/srcdb/yugbaytedb_test.go | 136 ++++++++++++++ yb-voyager/src/tgtdb/conn_pool_test.go | 46 +---- yb-voyager/src/tgtdb/main_test.go | 139 ++++++++++++++ yb-voyager/src/tgtdb/postgres_test.go | 91 +++++++-- yb-voyager/src/tgtdb/yugabytedb_test.go | 93 +++++++-- yb-voyager/test/containers/helpers.go | 60 ++++++ yb-voyager/test/containers/mysql_container.go | 149 +++++++++++++++ .../test/containers/oracle_container.go | 107 +++++++++++ .../test/containers/postgres_container.go | 151 +++++++++++++++ .../containers/test_schemas/mysql_schema.sql | 6 + .../containers/test_schemas/oracle_schema.sql | 47 +++++ .../test_schemas/postgresql_schema.sql | 1 + .../test_schemas/yugabytedb_schema.sql | 1 + yb-voyager/test/containers/testcontainers.go | 143 ++++++++++++++ .../test/containers/yugabytedb_container.go | 129 +++++++++++++ .../utils/testutils.go} | 56 +++++- yb-voyager/testcontainers/testcontainers.go | 108 ----------- 36 files changed, 1862 insertions(+), 210 deletions(-) create mode 100644 .github/workflows/integration-tests.yml create mode 100644 yb-voyager/src/srcdb/main_test.go create mode 100644 yb-voyager/src/srcdb/mysql_test.go create mode 100644 yb-voyager/src/srcdb/oracle_test.go create mode 100644 yb-voyager/src/srcdb/postgres_test.go create mode 100644 yb-voyager/src/srcdb/yugbaytedb_test.go create mode 100644 yb-voyager/src/tgtdb/main_test.go create mode 100644 yb-voyager/test/containers/helpers.go create mode 100644 yb-voyager/test/containers/mysql_container.go create mode 100644 yb-voyager/test/containers/oracle_container.go create mode 100644 yb-voyager/test/containers/postgres_container.go create mode 100644 yb-voyager/test/containers/test_schemas/mysql_schema.sql create mode 100644 yb-voyager/test/containers/test_schemas/oracle_schema.sql create mode 100644 yb-voyager/test/containers/test_schemas/postgresql_schema.sql create mode 100644 yb-voyager/test/containers/test_schemas/yugabytedb_schema.sql create mode 100644 yb-voyager/test/containers/testcontainers.go create mode 100644 yb-voyager/test/containers/yugabytedb_container.go rename yb-voyager/{src/testutils/testing_utils.go => test/utils/testutils.go} (89%) delete mode 100644 yb-voyager/testcontainers/testcontainers.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 96658551e..ea81fd4e7 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -8,7 +8,7 @@ on: jobs: - build: + build-and-test: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 @@ -26,7 +26,7 @@ jobs: - name: Test run: | cd yb-voyager - go test -v -skip '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... + go test -v -skip '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... -tags '!integration' - name: Vet run: | diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 000000000..b6b018be2 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,53 @@ +name: Go + +on: + push: + branches: ['main', '*.*-dev', '*.*.*-dev'] + pull_request: + branches: [main] + +env: + ORACLE_INSTANT_CLIENT_VERSION: "21.5.0.0.0-1" + +jobs: + integration-tests: + strategy: + fail-fast: false + + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: "1.23.1" + + - name: Build + run: | + cd yb-voyager + go build -v ./... + + # required by godror driver used in the tests + - name: Install Oracle Instant Clients + run: | + # Download and install the YB APT repository package + wget https://s3.us-west-2.amazonaws.com/downloads.yugabyte.com/repos/reporpms/yb-apt-repo_1.0.0_all.deb + sudo apt-get install -y ./yb-apt-repo_1.0.0_all.deb + sudo apt-get update -y + + # Install Oracle Instant Client packages using the defined version + sudo apt-get install -y oracle-instantclient-tools=${{ env.ORACLE_INSTANT_CLIENT_VERSION }} + sudo apt-get install -y oracle-instantclient-basic=${{ env.ORACLE_INSTANT_CLIENT_VERSION }} + sudo apt-get install -y oracle-instantclient-devel=${{ env.ORACLE_INSTANT_CLIENT_VERSION }} + sudo apt-get install -y oracle-instantclient-jdbc=${{ env.ORACLE_INSTANT_CLIENT_VERSION }} + sudo apt-get install -y oracle-instantclient-sqlplus=${{ env.ORACLE_INSTANT_CLIENT_VERSION }} + + # Clean up the YB APT repository package + sudo apt-get remove -y yb-apt-repo + rm -f yb-apt-repo_1.0.0_all.deb + + - name: Run Integration Tests + run: | + cd yb-voyager + go test -v -skip '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... -tags 'integration' diff --git a/.github/workflows/issue-tests.yml b/.github/workflows/issue-tests.yml index 2c965c542..7db2e6260 100644 --- a/.github/workflows/issue-tests.yml +++ b/.github/workflows/issue-tests.yml @@ -47,5 +47,4 @@ jobs: - name: Test Issues Against YB Version run: | cd yb-voyager - go test -v -run '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... - + go test -v -run '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... -tags '!integration' diff --git a/yb-voyager/cmd/common_test.go b/yb-voyager/cmd/common_test.go index d0046ea7a..6a5c46236 100644 --- a/yb-voyager/cmd/common_test.go +++ b/yb-voyager/cmd/common_test.go @@ -8,9 +8,9 @@ import ( "testing" "github.com/yugabyte/yb-voyager/yb-voyager/src/migassessment" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestAssessmentReportStructs(t *testing.T) { diff --git a/yb-voyager/cmd/exportDataStatus_test.go b/yb-voyager/cmd/exportDataStatus_test.go index beec85005..692710aac 100644 --- a/yb-voyager/cmd/exportDataStatus_test.go +++ b/yb-voyager/cmd/exportDataStatus_test.go @@ -6,8 +6,8 @@ import ( "reflect" "testing" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestExportSnapshotStatusStructs(t *testing.T) { diff --git a/yb-voyager/go.mod b/yb-voyager/go.mod index a8a306dbc..8cf7a703f 100644 --- a/yb-voyager/go.mod +++ b/yb-voyager/go.mod @@ -22,8 +22,8 @@ require ( github.com/gosuri/uilive v0.0.4 github.com/gosuri/uitable v0.0.4 github.com/hashicorp/go-version v1.7.0 - github.com/jackc/pgconn v1.13.0 - github.com/jackc/pgx/v4 v4.17.2 + github.com/jackc/pgconn v1.14.3 + github.com/jackc/pgx/v4 v4.18.3 github.com/jackc/pgx/v5 v5.0.3 github.com/lib/pq v1.10.9 github.com/mattn/go-sqlite3 v1.14.17 @@ -45,6 +45,7 @@ require ( golang.org/x/term v0.24.0 google.golang.org/api v0.169.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 + gotest.tools v2.2.0+incompatible ) require ( @@ -143,9 +144,9 @@ require ( github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pglogrepl v0.0.0-20231111135425-1627ab1b5780 github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgproto3/v2 v2.3.1 // indirect - github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect - github.com/jackc/pgtype v1.12.0 // indirect + github.com/jackc/pgproto3/v2 v2.3.3 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgtype v1.14.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/magiconair/properties v1.8.7 github.com/mattn/go-colorable v0.1.13 // indirect diff --git a/yb-voyager/go.sum b/yb-voyager/go.sum index 3b156d1c0..e68839fdc 100644 --- a/yb-voyager/go.sum +++ b/yb-voyager/go.sum @@ -1361,8 +1361,9 @@ github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsU github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.13.0 h1:3L1XMNV2Zvca/8BYhzcRFS70Lr0WlDg16Di6SFGAbys= github.com/jackc/pgconn v1.13.0/go.mod h1:AnowpAqO4CMIIJNZl2VJp+KrkAZciAkhEl0W0JIobpI= +github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= +github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pglogrepl v0.0.0-20231111135425-1627ab1b5780 h1:pNK2AKKIRC1MMMvpa6UiNtdtOebpiIloX7q2JZDkfsk= @@ -1380,22 +1381,26 @@ github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvW github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.3.1 h1:nwj7qwf0S+Q7ISFfBndqeLwSwxs+4DPsbRFjECT1Y4Y= github.com/jackc/pgproto3/v2 v2.3.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= +github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.12.0 h1:Dlq8Qvcch7kiehm8wPGIW0W3KsCCHJnRacKW0UM8n5w= github.com/jackc/pgtype v1.12.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= +github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.17.2 h1:0Ut0rpeKwvIVbMQ1KbMBU4h6wxehBI535LK6Flheh8E= github.com/jackc/pgx/v4 v4.17.2/go.mod h1:lcxIZN44yMIrWI78a5CpucdD14hX0SBDbNRvjDBItsw= +github.com/jackc/pgx/v4 v4.18.3 h1:dE2/TrEsGX3RBprb3qryqSV9Y60iZN1C6i8IrmW9/BA= +github.com/jackc/pgx/v4 v4.18.3/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= github.com/jackc/pgx/v5 v5.0.3 h1:4flM5ecR/555F0EcnjdaZa6MhBU+nr0QbZIo5vaKjuM= github.com/jackc/pgx/v5 v5.0.3/go.mod h1:JBbvW3Hdw77jKl9uJrEDATUZIFM2VFPzRq4RWIhkF4o= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= diff --git a/yb-voyager/src/callhome/diagnostics_test.go b/yb-voyager/src/callhome/diagnostics_test.go index a212004a0..9d63b562b 100644 --- a/yb-voyager/src/callhome/diagnostics_test.go +++ b/yb-voyager/src/callhome/diagnostics_test.go @@ -5,8 +5,8 @@ import ( "testing" "github.com/google/uuid" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestCallhomeStructs(t *testing.T) { diff --git a/yb-voyager/src/cp/yugabyted/yugabyted_test.go b/yb-voyager/src/cp/yugabyted/yugabyted_test.go index 1c6690999..90bf0c5e2 100644 --- a/yb-voyager/src/cp/yugabyted/yugabyted_test.go +++ b/yb-voyager/src/cp/yugabyted/yugabyted_test.go @@ -14,29 +14,33 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v4/pgxpool" - _ "github.com/lib/pq" // PostgreSQL driver + _ "github.com/jackc/pgx/v5/stdlib" "github.com/stretchr/testify/assert" controlPlane "github.com/yugabyte/yb-voyager/yb-voyager/src/cp" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" - "github.com/yugabyte/yb-voyager/yb-voyager/testcontainers" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" + testcontainers "github.com/yugabyte/yb-voyager/yb-voyager/test/containers" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestYugabyteDTableSchema(t *testing.T) { ctx := context.Background() - // Start a YugabyteDB container - ybContainer, host, port, err := testcontainers.StartDBContainer(ctx, testcontainers.YUGABYTEDB) + yugabyteDBContainer := testcontainers.NewTestContainer("yugabytedb", nil) + err := yugabyteDBContainer.Start(ctx) + if err != nil { + utils.ErrExit("Failed to start yugabytedb container: %v", err) + } + defer testcontainers.TerminateAllContainers() assert.NoError(t, err, "Failed to start YugabyteDB container") - defer ybContainer.Terminate(ctx) // Connect to the database - dsn := fmt.Sprintf("host=%s port=%s user=yugabyte password=yugabyte dbname=yugabyte sslmode=disable", host, port.Port()) - db, err := sql.Open("postgres", dsn) + dsn := yugabyteDBContainer.GetConnectionString() + db, err := sql.Open("pgx", dsn) assert.NoError(t, err) defer db.Close() // Wait for the database to be ready - err = testcontainers.WaitForDBToBeReady(db) + err = testutils.WaitForDBToBeReady(db) assert.NoError(t, err) // Export the database connection string to env variable YUGABYTED_DB_CONN_STRING err = os.Setenv("YUGABYTED_DB_CONN_STRING", dsn) diff --git a/yb-voyager/src/datafile/descriptor_test.go b/yb-voyager/src/datafile/descriptor_test.go index dc617d090..6092aba4c 100644 --- a/yb-voyager/src/datafile/descriptor_test.go +++ b/yb-voyager/src/datafile/descriptor_test.go @@ -6,7 +6,7 @@ import ( "reflect" "testing" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestDescriptorStructs(t *testing.T) { diff --git a/yb-voyager/src/dbzm/status_test.go b/yb-voyager/src/dbzm/status_test.go index 9b384e9ad..753188dfd 100644 --- a/yb-voyager/src/dbzm/status_test.go +++ b/yb-voyager/src/dbzm/status_test.go @@ -4,7 +4,7 @@ import ( "reflect" "testing" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestExportStatusStructs(t *testing.T) { diff --git a/yb-voyager/src/metadb/metadataDB_test.go b/yb-voyager/src/metadb/metadataDB_test.go index 89648a26b..b6c012692 100644 --- a/yb-voyager/src/metadb/metadataDB_test.go +++ b/yb-voyager/src/metadb/metadataDB_test.go @@ -7,7 +7,7 @@ import ( "testing" _ "github.com/mattn/go-sqlite3" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) // Test the initMetaDB function diff --git a/yb-voyager/src/migassessment/assessmentDB_test.go b/yb-voyager/src/migassessment/assessmentDB_test.go index 843d1cca0..854b52c8f 100644 --- a/yb-voyager/src/migassessment/assessmentDB_test.go +++ b/yb-voyager/src/migassessment/assessmentDB_test.go @@ -7,7 +7,7 @@ import ( "testing" _ "github.com/mattn/go-sqlite3" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestInitAssessmentDB(t *testing.T) { diff --git a/yb-voyager/src/namereg/namereg_test.go b/yb-voyager/src/namereg/namereg_test.go index 9d6b16c52..c785dbff0 100644 --- a/yb-voyager/src/namereg/namereg_test.go +++ b/yb-voyager/src/namereg/namereg_test.go @@ -13,8 +13,8 @@ import ( "github.com/stretchr/testify/require" "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) var oracleToYBNameRegistry = &NameRegistry{ diff --git a/yb-voyager/src/srcdb/main_test.go b/yb-voyager/src/srcdb/main_test.go new file mode 100644 index 000000000..770855ad6 --- /dev/null +++ b/yb-voyager/src/srcdb/main_test.go @@ -0,0 +1,176 @@ +//go:build integration + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package srcdb + +import ( + "context" + "os" + "testing" + + _ "github.com/godror/godror" + _ "github.com/jackc/pgx/v5/stdlib" + log "github.com/sirupsen/logrus" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" + testcontainers "github.com/yugabyte/yb-voyager/yb-voyager/test/containers" +) + +type TestDB struct { + testcontainers.TestContainer + *Source +} + +var ( + testPostgresSource *TestDB + testOracleSource *TestDB + testMySQLSource *TestDB + testYugabyteDBSource *TestDB +) + +func TestMain(m *testing.M) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + postgresContainer := testcontainers.NewTestContainer("postgresql", nil) + err := postgresContainer.Start(ctx) + if err != nil { + utils.ErrExit("Failed to start postgres container: %v", err) + } + host, port, err := postgresContainer.GetHostPort() + if err != nil { + utils.ErrExit("%v", err) + } + testPostgresSource = &TestDB{ + TestContainer: postgresContainer, + Source: &Source{ + DBType: "postgresql", + DBVersion: postgresContainer.GetConfig().DBVersion, + User: postgresContainer.GetConfig().User, + Password: postgresContainer.GetConfig().Password, + Schema: postgresContainer.GetConfig().Schema, + DBName: postgresContainer.GetConfig().DBName, + Host: host, + Port: port, + SSLMode: "disable", + }, + } + err = testPostgresSource.DB().Connect() + if err != nil { + utils.ErrExit("Failed to connect to postgres database: %w", err) + } + defer testPostgresSource.DB().Disconnect() + + oracleContainer := testcontainers.NewTestContainer("oracle", nil) + err = oracleContainer.Start(ctx) + if err != nil { + utils.ErrExit("Failed to start oracle container: %v", err) + } + host, port, err = oracleContainer.GetHostPort() + if err != nil { + utils.ErrExit("%v", err) + } + + testOracleSource = &TestDB{ + TestContainer: oracleContainer, + Source: &Source{ + DBType: "oracle", + DBVersion: oracleContainer.GetConfig().DBVersion, + User: oracleContainer.GetConfig().User, + Password: oracleContainer.GetConfig().Password, + Schema: oracleContainer.GetConfig().Schema, + DBName: oracleContainer.GetConfig().DBName, + Host: host, + Port: port, + }, + } + + err = testOracleSource.DB().Connect() + if err != nil { + utils.ErrExit("Failed to connect to oracle database: %w", err) + } + defer testOracleSource.DB().Disconnect() + + mysqlContainer := testcontainers.NewTestContainer("mysql", nil) + err = mysqlContainer.Start(ctx) + if err != nil { + utils.ErrExit("Failed to start mysql container: %v", err) + } + host, port, err = mysqlContainer.GetHostPort() + if err != nil { + utils.ErrExit("%v", err) + } + testMySQLSource = &TestDB{ + TestContainer: mysqlContainer, + Source: &Source{ + DBType: "mysql", + DBVersion: mysqlContainer.GetConfig().DBVersion, + User: mysqlContainer.GetConfig().User, + Password: mysqlContainer.GetConfig().Password, + Schema: mysqlContainer.GetConfig().Schema, + DBName: mysqlContainer.GetConfig().DBName, + Host: host, + Port: port, + SSLMode: "disable", + }, + } + + err = testMySQLSource.DB().Connect() + if err != nil { + utils.ErrExit("Failed to connect to mysql database: %w", err) + } + defer testMySQLSource.DB().Disconnect() + + yugabytedbContainer := testcontainers.NewTestContainer("yugabytedb", nil) + err = yugabytedbContainer.Start(ctx) + if err != nil { + utils.ErrExit("Failed to start yugabytedb container: %v", err) + } + host, port, err = yugabytedbContainer.GetHostPort() + if err != nil { + utils.ErrExit("%v", err) + } + testYugabyteDBSource = &TestDB{ + TestContainer: yugabytedbContainer, + Source: &Source{ + DBType: "yugabytedb", + DBVersion: yugabytedbContainer.GetConfig().DBVersion, + User: yugabytedbContainer.GetConfig().User, + Password: yugabytedbContainer.GetConfig().Password, + Schema: yugabytedbContainer.GetConfig().Schema, + DBName: yugabytedbContainer.GetConfig().DBName, + Host: host, + Port: port, + SSLMode: "disable", + }, + } + + err = testYugabyteDBSource.DB().Connect() + if err != nil { + utils.ErrExit("Failed to connect to yugabytedb database: %w", err) + } + defer testYugabyteDBSource.DB().Disconnect() + + // to avoid info level logs flooding the test output + log.SetLevel(log.WarnLevel) + + exitCode := m.Run() + + // cleanig up all the running containers + testcontainers.TerminateAllContainers() + + os.Exit(exitCode) +} diff --git a/yb-voyager/src/srcdb/mysql_test.go b/yb-voyager/src/srcdb/mysql_test.go new file mode 100644 index 000000000..db7472416 --- /dev/null +++ b/yb-voyager/src/srcdb/mysql_test.go @@ -0,0 +1,89 @@ +//go:build integration + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package srcdb + +import ( + "testing" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" + "gotest.tools/assert" +) + +func TestMysqlGetAllTableNames(t *testing.T) { + testMySQLSource.ExecuteSqls( + `CREATE DATABASE test;`, + `CREATE TABLE test.foo ( + id INT PRIMARY KEY, + name VARCHAR(255) + );`, + `CREATE TABLE test.bar ( + id INT PRIMARY KEY, + name VARCHAR(255) + );`, + `CREATE TABLE test.non_pk1( + id INT, + name VARCHAR(255) + );`) + defer testMySQLSource.ExecuteSqls(`DROP DATABASE test;`) + + sqlname.SourceDBType = "mysql" + testMySQLSource.Source.DBName = "test" // used in query of GetAllTableNames() + + // Test GetAllTableNames + actualTables := testMySQLSource.DB().GetAllTableNames() + expectedTables := []*sqlname.SourceName{ + sqlname.NewSourceName("test", "foo"), + sqlname.NewSourceName("test", "bar"), + sqlname.NewSourceName("test", "non_pk1"), + } + assert.Equal(t, len(expectedTables), len(actualTables), "Expected number of tables to match") + + testutils.AssertEqualSourceNameSlices(t, expectedTables, actualTables) +} + +// TODO: Seems like a Bug somwhere, because now mysql.GetAllNonPkTables() as it is returning all the tables created in this test +// func TestMySQLGetNonPKTables(t *testing.T) { +// testMySQLSource.ExecuteSqls( +// `CREATE DATABASE test;`, +// `CREATE TABLE test.table1 ( +// id INT AUTO_INCREMENT PRIMARY KEY, +// name VARCHAR(100) +// );`, +// `CREATE TABLE test.table2 ( +// id INT AUTO_INCREMENT PRIMARY KEY, +// email VARCHAR(100) +// );`, +// `CREATE TABLE test.non_pk1( +// id INT, +// name VARCHAR(255) +// );`, +// `CREATE TABLE test.non_pk2( +// id INT, +// name VARCHAR(255) +// );`) +// defer testMySQLSource.ExecuteSqls(`DROP DATABASE test;`) + +// testMySQLSource.Source.DBName = "test" +// actualTables, err := testMySQLSource.DB().GetNonPKTables() +// assert.NilError(t, err, "Expected nil but non nil error: %v", err) + +// expectedTables := []string{"test.non_pk1", "test.non_pk2"} + +// testutils.AssertEqualStringSlices(t, expectedTables, actualTables) +// } diff --git a/yb-voyager/src/srcdb/oracle_test.go b/yb-voyager/src/srcdb/oracle_test.go new file mode 100644 index 000000000..9369b8257 --- /dev/null +++ b/yb-voyager/src/srcdb/oracle_test.go @@ -0,0 +1,80 @@ +//go:build integration + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package srcdb + +import ( + "testing" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" + "gotest.tools/assert" +) + +func TestOracleGetAllTableNames(t *testing.T) { + sqlname.SourceDBType = "oracle" + + // Test GetAllTableNames + actualTables := testOracleSource.DB().GetAllTableNames() + expectedTables := []*sqlname.SourceName{ + sqlname.NewSourceName("YBVOYAGER", "foo"), + sqlname.NewSourceName("YBVOYAGER", "bar"), + sqlname.NewSourceName("YBVOYAGER", "table1"), + sqlname.NewSourceName("YBVOYAGER", "table2"), + sqlname.NewSourceName("YBVOYAGER", "unique_table"), + sqlname.NewSourceName("YBVOYAGER", "non_pk1"), + sqlname.NewSourceName("YBVOYAGER", "non_pk2"), + } + assert.Equal(t, len(expectedTables), len(actualTables), "Expected number of tables to match") + + testutils.AssertEqualSourceNameSlices(t, expectedTables, actualTables) +} + +func TestOracleGetTableToUniqueKeyColumnsMap(t *testing.T) { + objectName := sqlname.NewObjectName("oracle", "YBVOYAGER", "YBVOYAGER", "UNIQUE_TABLE") + + // Test GetTableToUniqueKeyColumnsMap + tableList := []sqlname.NameTuple{ + {CurrentName: objectName}, + } + uniqueKeys, err := testOracleSource.DB().GetTableToUniqueKeyColumnsMap(tableList) + if err != nil { + t.Fatalf("Error retrieving unique keys: %v", err) + } + + expectedKeys := map[string][]string{ + "UNIQUE_TABLE": {"EMAIL", "PHONE", "ADDRESS"}, + } + + // Compare the maps by iterating over each table and asserting the columns list + for table, expectedColumns := range expectedKeys { + actualColumns, exists := uniqueKeys[table] + if !exists { + t.Errorf("Expected table %s not found in uniqueKeys", table) + } + + testutils.AssertEqualStringSlices(t, expectedColumns, actualColumns) + } +} + +func TestOracleGetNonPKTables(t *testing.T) { + actualTables, err := testOracleSource.DB().GetNonPKTables() + assert.NilError(t, err, "Expected nil but non nil error: %v", err) + + expectedTables := []string{`YBVOYAGER."NON_PK1"`, `YBVOYAGER."NON_PK2"`} + testutils.AssertEqualStringSlices(t, expectedTables, actualTables) +} diff --git a/yb-voyager/src/srcdb/postgres.go b/yb-voyager/src/srcdb/postgres.go index 4c90ffb5e..ba2e21b72 100644 --- a/yb-voyager/src/srcdb/postgres.go +++ b/yb-voyager/src/srcdb/postgres.go @@ -940,6 +940,7 @@ var PG_QUERY_TO_CHECK_IF_TABLE_HAS_PK = `SELECT nspname AS schema_name, relname FROM pg_class c LEFT JOIN pg_namespace n ON n.oid = c.relnamespace LEFT JOIN pg_constraint con ON con.conrelid = c.oid AND con.contype = 'p' +WHERE c.relkind = 'r' OR c.relkind = 'p' -- Only consider table objects GROUP BY schema_name, table_name HAVING nspname IN (%s);` func (pg *PostgreSQL) GetNonPKTables() ([]string, error) { @@ -964,8 +965,9 @@ func (pg *PostgreSQL) GetNonPKTables() ([]string, error) { if err != nil { return nil, fmt.Errorf("error in scanning query rows for primary key: %v", err) } - table := sqlname.NewSourceName(schemaName, fmt.Sprintf(`"%s"`, tableName)) + if pkCount == 0 { + table := sqlname.NewSourceName(schemaName, fmt.Sprintf(`"%s"`, tableName)) nonPKTables = append(nonPKTables, table.Qualified.Quoted) } } diff --git a/yb-voyager/src/srcdb/postgres_test.go b/yb-voyager/src/srcdb/postgres_test.go new file mode 100644 index 000000000..41ac55d5a --- /dev/null +++ b/yb-voyager/src/srcdb/postgres_test.go @@ -0,0 +1,136 @@ +//go:build integration + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package srcdb + +import ( + "testing" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" + "gotest.tools/assert" +) + +func TestPostgresGetAllTableNames(t *testing.T) { + testPostgresSource.TestContainer.ExecuteSqls( + `CREATE SCHEMA test_schema;`, + `CREATE TABLE test_schema.foo ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.foo values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.bar ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.bar values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.non_pk1( + id INT, + name VARCHAR(255) + );`) + defer testPostgresSource.TestContainer.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + sqlname.SourceDBType = "postgresql" + testPostgresSource.Source.Schema = "test_schema" // used in query of GetAllTableNames() + + // Test GetAllTableNames + actualTables := testPostgresSource.DB().GetAllTableNames() + expectedTables := []*sqlname.SourceName{ + sqlname.NewSourceName("test_schema", "foo"), + sqlname.NewSourceName("test_schema", "bar"), + sqlname.NewSourceName("test_schema", "non_pk1"), + } + assert.Equal(t, len(expectedTables), len(actualTables), "Expected number of tables to match") + testutils.AssertEqualSourceNameSlices(t, expectedTables, actualTables) +} + +func TestPostgresGetTableToUniqueKeyColumnsMap(t *testing.T) { + testPostgresSource.TestContainer.ExecuteSqls( + `CREATE SCHEMA test_schema;`, + `CREATE TABLE test_schema.unique_table ( + id SERIAL PRIMARY KEY, + email VARCHAR(255) UNIQUE, + phone VARCHAR(20) UNIQUE, + address VARCHAR(255) UNIQUE + );`, + `INSERT INTO test_schema.unique_table (email, phone, address) VALUES + ('john@example.com', '1234567890', '123 Elm Street'), + ('jane@example.com', '0987654321', '456 Oak Avenue');`, + `CREATE TABLE test_schema.another_unique_table ( + user_id SERIAL PRIMARY KEY, + username VARCHAR(50) UNIQUE, + age INT + );`, + `CREATE UNIQUE INDEX idx_age ON test_schema.another_unique_table(age);`, + `INSERT INTO test_schema.another_unique_table (username, age) VALUES + ('user1', 30), + ('user2', 40);`) + defer testPostgresSource.TestContainer.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + uniqueTablesList := []sqlname.NameTuple{ + {CurrentName: sqlname.NewObjectName("postgresql", "test_schema", "test_schema", "unique_table")}, + {CurrentName: sqlname.NewObjectName("postgresql", "test_schema", "test_schema", "another_unique_table")}, + } + + actualUniqKeys, err := testPostgresSource.DB().GetTableToUniqueKeyColumnsMap(uniqueTablesList) + if err != nil { + t.Fatalf("Error retrieving unique keys: %v", err) + } + + expectedUniqKeys := map[string][]string{ + "test_schema.unique_table": {"email", "phone", "address"}, + "test_schema.another_unique_table": {"username", "age"}, + } + + // Compare the maps by iterating over each table and asserting the columns list + for table, expectedColumns := range expectedUniqKeys { + actualColumns, exists := actualUniqKeys[table] + if !exists { + t.Errorf("Expected table %s not found in uniqueKeys", table) + } + + testutils.AssertEqualStringSlices(t, expectedColumns, actualColumns) + } +} + +func TestPostgresGetNonPKTables(t *testing.T) { + testPostgresSource.TestContainer.ExecuteSqls( + `CREATE SCHEMA test_schema;`, + `CREATE TABLE test_schema.table1 ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) + );`, + `CREATE TABLE test_schema.table2 ( + id SERIAL PRIMARY KEY, + email VARCHAR(100) + );`, + `CREATE TABLE test_schema.non_pk1( + id INT, + name VARCHAR(255) + );`, + `CREATE TABLE test_schema.non_pk2( + id INT, + name VARCHAR(255) + );`) + defer testPostgresSource.TestContainer.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + actualTables, err := testPostgresSource.DB().GetNonPKTables() + assert.NilError(t, err, "Expected nil but non nil error: %v", err) + + expectedTables := []string{`test_schema."non_pk2"`, `test_schema."non_pk1"`} // func returns table.Qualified.Quoted + testutils.AssertEqualStringSlices(t, expectedTables, actualTables) +} diff --git a/yb-voyager/src/srcdb/yugbaytedb_test.go b/yb-voyager/src/srcdb/yugbaytedb_test.go new file mode 100644 index 000000000..4a7cf8e6f --- /dev/null +++ b/yb-voyager/src/srcdb/yugbaytedb_test.go @@ -0,0 +1,136 @@ +//go:build integration + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package srcdb + +import ( + "testing" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" + "gotest.tools/assert" +) + +func TestYugabyteGetAllTableNames(t *testing.T) { + testYugabyteDBSource.TestContainer.ExecuteSqls( + `CREATE SCHEMA test_schema;`, + `CREATE TABLE test_schema.foo ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.foo values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.bar ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.bar values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.non_pk1( + id INT, + name VARCHAR(255) + );`) + defer testYugabyteDBSource.TestContainer.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + sqlname.SourceDBType = "postgresql" + testYugabyteDBSource.Source.Schema = "test_schema" + + // Test GetAllTableNames + actualTables := testYugabyteDBSource.DB().GetAllTableNames() + expectedTables := []*sqlname.SourceName{ + sqlname.NewSourceName("test_schema", "foo"), + sqlname.NewSourceName("test_schema", "bar"), + sqlname.NewSourceName("test_schema", "non_pk1"), + } + assert.Equal(t, len(expectedTables), len(actualTables), "Expected number of tables to match") + testutils.AssertEqualSourceNameSlices(t, expectedTables, actualTables) +} + +func TestYugabyteGetTableToUniqueKeyColumnsMap(t *testing.T) { + testYugabyteDBSource.TestContainer.ExecuteSqls( + `CREATE SCHEMA test_schema;`, + `CREATE TABLE test_schema.unique_table ( + id SERIAL PRIMARY KEY, + email VARCHAR(255) UNIQUE, + phone VARCHAR(20) UNIQUE, + address VARCHAR(255) UNIQUE + );`, + `INSERT INTO test_schema.unique_table (email, phone, address) VALUES + ('john@example.com', '1234567890', '123 Elm Street'), + ('jane@example.com', '0987654321', '456 Oak Avenue');`, + `CREATE TABLE test_schema.another_unique_table ( + user_id SERIAL PRIMARY KEY, + username VARCHAR(50) UNIQUE, + age INT + );`, + `CREATE UNIQUE INDEX idx_age ON test_schema.another_unique_table(age);`, + `INSERT INTO test_schema.another_unique_table (username, age) VALUES + ('user1', 30), + ('user2', 40);`) + defer testYugabyteDBSource.TestContainer.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + uniqueTablesList := []sqlname.NameTuple{ + {CurrentName: sqlname.NewObjectName("postgresql", "test_schema", "test_schema", "unique_table")}, + {CurrentName: sqlname.NewObjectName("postgresql", "test_schema", "test_schema", "another_unique_table")}, + } + + actualUniqKeys, err := testYugabyteDBSource.DB().GetTableToUniqueKeyColumnsMap(uniqueTablesList) + if err != nil { + t.Fatalf("Error retrieving unique keys: %v", err) + } + + expectedUniqKeys := map[string][]string{ + "test_schema.unique_table": {"email", "phone", "address"}, + "test_schema.another_unique_table": {"username", "age"}, + } + + // Compare the maps by iterating over each table and asserting the columns list + for table, expectedColumns := range expectedUniqKeys { + actualColumns, exists := actualUniqKeys[table] + if !exists { + t.Errorf("Expected table %s not found in uniqueKeys", table) + } + + testutils.AssertEqualStringSlices(t, expectedColumns, actualColumns) + } +} + +func TestYugabyteGetNonPKTables(t *testing.T) { + testYugabyteDBSource.TestContainer.ExecuteSqls( + `CREATE SCHEMA test_schema;`, + `CREATE TABLE test_schema.table1 ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) + );`, + `CREATE TABLE test_schema.table2 ( + id SERIAL PRIMARY KEY, + email VARCHAR(100) + );`, + `CREATE TABLE test_schema.non_pk1( + id INT, + name VARCHAR(255) + );`, + `CREATE TABLE test_schema.non_pk2( + id INT, + name VARCHAR(255) + );`) + defer testYugabyteDBSource.TestContainer.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + actualTables, err := testYugabyteDBSource.DB().GetNonPKTables() + assert.NilError(t, err, "Expected nil but non nil error: %v", err) + + expectedTables := []string{`test_schema."non_pk2"`, `test_schema."non_pk1"`} // func returns table.Qualified.Quoted + testutils.AssertEqualStringSlices(t, expectedTables, actualTables) +} diff --git a/yb-voyager/src/tgtdb/conn_pool_test.go b/yb-voyager/src/tgtdb/conn_pool_test.go index 59fd4dad4..7d4ce4380 100644 --- a/yb-voyager/src/tgtdb/conn_pool_test.go +++ b/yb-voyager/src/tgtdb/conn_pool_test.go @@ -23,44 +23,18 @@ import ( "time" "github.com/jackc/pgx/v4" + log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" - - embeddedpostgres "github.com/fergusstrange/embedded-postgres" ) -// var postgres *embeddedpostgres.EmbeddedPostgres - -func setupPostgres(t *testing.T) *embeddedpostgres.EmbeddedPostgres { - postgres := embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig(). - Username("postgres"). - Password("postgres"). - Database("test"). - Port(9876). - StartTimeout(30 * time.Second)) - err := postgres.Start() - if err != nil { - t.Fatal(err) - } - return postgres -} - -func shutdownPostgres(postgres *embeddedpostgres.EmbeddedPostgres, t *testing.T) { - err := postgres.Stop() - if err != nil { - t.Fatal(err) - } -} - func TestBasic(t *testing.T) { - postgres := setupPostgres(t) - defer shutdownPostgres(postgres, t) // GIVEN: a conn pool of size 10. size := 10 connParams := &ConnectionParams{ NumConnections: size, NumMaxConnections: size, - ConnUriList: []string{fmt.Sprintf("postgresql://postgres:postgres@localhost:%d/test", 9876)}, + ConnUriList: []string{testYugabyteDBTarget.GetConnectionString()}, SessionInitScript: []string{}, } pool := NewConnectionPool(connParams) @@ -88,8 +62,6 @@ func dummyProcess(pool *ConnectionPool, milliseconds int, wg *sync.WaitGroup) { } func TestIncreaseConnectionsUptoMax(t *testing.T) { - postgres := setupPostgres(t) - defer shutdownPostgres(postgres, t) // GIVEN: a conn pool of size 10, with max 20 connections. size := 10 maxSize := 20 @@ -97,7 +69,7 @@ func TestIncreaseConnectionsUptoMax(t *testing.T) { connParams := &ConnectionParams{ NumConnections: size, NumMaxConnections: maxSize, - ConnUriList: []string{fmt.Sprintf("postgresql://postgres:postgres@localhost:%d/test", 9876)}, + ConnUriList: []string{testYugabyteDBTarget.GetConnectionString()}, SessionInitScript: []string{}, } pool := NewConnectionPool(connParams) @@ -105,7 +77,7 @@ func TestIncreaseConnectionsUptoMax(t *testing.T) { // WHEN: multiple goroutines acquire connection, perform some operation // and release connection back to pool - // WHEN: we keep increasing the connnections upto the max.. + // WHEN: we keep increasing the connections upto the max.. var wg sync.WaitGroup wg.Add(1) @@ -133,8 +105,6 @@ func TestIncreaseConnectionsUptoMax(t *testing.T) { } func TestDecreaseConnectionsUptoMin(t *testing.T) { - postgres := setupPostgres(t) - defer shutdownPostgres(postgres, t) // GIVEN: a conn pool of size 10, with max 20 connections. size := 10 maxSize := 20 @@ -142,7 +112,7 @@ func TestDecreaseConnectionsUptoMin(t *testing.T) { connParams := &ConnectionParams{ NumConnections: size, NumMaxConnections: maxSize, - ConnUriList: []string{fmt.Sprintf("postgresql://postgres:postgres@localhost:%d/test", 9876)}, + ConnUriList: []string{testYugabyteDBTarget.GetConnectionString()}, SessionInitScript: []string{}, } pool := NewConnectionPool(connParams) @@ -178,8 +148,6 @@ func TestDecreaseConnectionsUptoMin(t *testing.T) { } func TestUpdateConnectionsRandom(t *testing.T) { - postgres := setupPostgres(t) - defer shutdownPostgres(postgres, t) // GIVEN: a conn pool of size 10, with max 20 connections. size := 10 maxSize := 20 @@ -187,7 +155,7 @@ func TestUpdateConnectionsRandom(t *testing.T) { connParams := &ConnectionParams{ NumConnections: size, NumMaxConnections: maxSize, - ConnUriList: []string{fmt.Sprintf("postgresql://postgres:postgres@localhost:%d/test", 9876)}, + ConnUriList: []string{testYugabyteDBTarget.GetConnectionString()}, SessionInitScript: []string{}, } pool := NewConnectionPool(connParams) @@ -207,7 +175,7 @@ func TestUpdateConnectionsRandom(t *testing.T) { if pool.size+randomNumber < 1 || (pool.size+randomNumber > pool.params.NumMaxConnections) { continue } - fmt.Printf("i=%d, updating by %d. New pool size expected = %d\n", i, randomNumber, *expectedFinalSize+randomNumber) + log.Infof("i=%d, updating by %d. New pool size expected = %d\n", i, randomNumber, *expectedFinalSize+randomNumber) err := pool.UpdateNumConnections(randomNumber) assert.NoError(t, err) time.Sleep(10 * time.Millisecond) diff --git a/yb-voyager/src/tgtdb/main_test.go b/yb-voyager/src/tgtdb/main_test.go new file mode 100644 index 000000000..46cc2150e --- /dev/null +++ b/yb-voyager/src/tgtdb/main_test.go @@ -0,0 +1,139 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package tgtdb + +import ( + "context" + "os" + "testing" + + _ "github.com/godror/godror" + _ "github.com/jackc/pgx/v5/stdlib" + log "github.com/sirupsen/logrus" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" + testcontainers "github.com/yugabyte/yb-voyager/yb-voyager/test/containers" +) + +type TestDB struct { + testcontainers.TestContainer + TargetDB +} + +var ( + testPostgresTarget *TestDB + testOracleTarget *TestDB + testYugabyteDBTarget *TestDB +) + +func TestMain(m *testing.M) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + postgresContainer := testcontainers.NewTestContainer("postgresql", nil) + err := postgresContainer.Start(ctx) + if err != nil { + utils.ErrExit("Failed to start postgres container: %v", err) + } + host, port, err := postgresContainer.GetHostPort() + if err != nil { + utils.ErrExit("%v", err) + } + testPostgresTarget = &TestDB{ + TestContainer: postgresContainer, + TargetDB: NewTargetDB(&TargetConf{ + TargetDBType: "postgresql", + DBVersion: postgresContainer.GetConfig().DBVersion, + User: postgresContainer.GetConfig().User, + Password: postgresContainer.GetConfig().Password, + Schema: postgresContainer.GetConfig().Schema, + DBName: postgresContainer.GetConfig().DBName, + Host: host, + Port: port, + SSLMode: "disable", + }), + } + + err = testPostgresTarget.Init() + if err != nil { + utils.ErrExit("Failed to connect to postgres database: %w", err) + } + defer testPostgresTarget.Finalize() + + // oracleContainer := testcontainers.NewTestContainer("oracle", nil) + // _ = oracleContainer.Start(ctx) + // host, port, err = oracleContainer.GetHostPort() + // if err != nil { + // utils.ErrExit("%v", err) + // } + // testOracleTarget = &TestDB2{ + // Container: oracleContainer, + // TargetDB: NewTargetDB(&TargetConf{ + // TargetDBType: "oracle", + // DBVersion: oracleContainer.GetConfig().DBVersion, + // User: oracleContainer.GetConfig().User, + // Password: oracleContainer.GetConfig().Password, + // Schema: oracleContainer.GetConfig().Schema, + // DBName: oracleContainer.GetConfig().DBName, + // Host: host, + // Port: port, + // }), + // } + + // err = testOracleTarget.Init() + // if err != nil { + // utils.ErrExit("Failed to connect to oracle database: %w", err) + // } + // defer testOracleTarget.Finalize() + + yugabytedbContainer := testcontainers.NewTestContainer("yugabytedb", nil) + err = yugabytedbContainer.Start(ctx) + if err != nil { + utils.ErrExit("Failed to start yugabytedb container: %v", err) + } + host, port, err = yugabytedbContainer.GetHostPort() + if err != nil { + utils.ErrExit("%v", err) + } + testYugabyteDBTarget = &TestDB{ + TestContainer: yugabytedbContainer, + TargetDB: NewTargetDB(&TargetConf{ + TargetDBType: "yugabytedb", + DBVersion: yugabytedbContainer.GetConfig().DBVersion, + User: yugabytedbContainer.GetConfig().User, + Password: yugabytedbContainer.GetConfig().Password, + Schema: yugabytedbContainer.GetConfig().Schema, + DBName: yugabytedbContainer.GetConfig().DBName, + Host: host, + Port: port, + }), + } + + err = testYugabyteDBTarget.Init() + if err != nil { + utils.ErrExit("Failed to connect to yugabytedb database: %w", err) + } + defer testYugabyteDBTarget.Finalize() + + // to avoid info level logs flooding the test output + log.SetLevel(log.WarnLevel) + + exitCode := m.Run() + + // cleaning up all the running containers + testcontainers.TerminateAllContainers() + + os.Exit(exitCode) +} diff --git a/yb-voyager/src/tgtdb/postgres_test.go b/yb-voyager/src/tgtdb/postgres_test.go index 251ced5a6..e0e9bd60c 100644 --- a/yb-voyager/src/tgtdb/postgres_test.go +++ b/yb-voyager/src/tgtdb/postgres_test.go @@ -1,33 +1,39 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package tgtdb import ( - "context" "database/sql" "fmt" "strings" "testing" + _ "github.com/jackc/pgx/v5/stdlib" "github.com/stretchr/testify/assert" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" - "github.com/yugabyte/yb-voyager/yb-voyager/testcontainers" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestCreateVoyagerSchemaPG(t *testing.T) { - ctx := context.Background() - - // Start a PostgreSQL container - pgContainer, host, port, err := testcontainers.StartDBContainer(ctx, testcontainers.POSTGRESQL) - assert.NoError(t, err, "Failed to start PostgreSQL container") - defer pgContainer.Terminate(ctx) - - // Connect to the database - dsn := fmt.Sprintf("host=%s port=%s user=testuser password=testpassword dbname=testdb sslmode=disable", host, port.Port()) - db, err := sql.Open("postgres", dsn) + db, err := sql.Open("pgx", testPostgresTarget.GetConnectionString()) assert.NoError(t, err) defer db.Close() // Wait for the database to be ready - err = testcontainers.WaitForDBToBeReady(db) + err = testutils.WaitForDBToBeReady(db) assert.NoError(t, err) // Initialize the TargetYugabyteDB instance @@ -80,3 +86,60 @@ func TestCreateVoyagerSchemaPG(t *testing.T) { }) } } + +func TestPostgresGetNonEmptyTables(t *testing.T) { + testPostgresTarget.ExecuteSqls( + `CREATE SCHEMA test_schema`, + `CREATE TABLE test_schema.foo ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.foo values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.bar ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.bar values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.unique_table ( + id SERIAL PRIMARY KEY, + email VARCHAR(100), + phone VARCHAR(100), + address VARCHAR(255), + UNIQUE (email, phone) -- Unique constraint on combination of columns + );`, + `CREATE TABLE test_schema.table1 ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) + );`, + `CREATE TABLE test_schema.table2 ( + id SERIAL PRIMARY KEY, + email VARCHAR(100) + );`, + `CREATE TABLE test_schema.non_pk1( + id INT, + name VARCHAR(255) + );`, + `CREATE TABLE test_schema.non_pk2( + id INT, + name VARCHAR(255) + );`) + defer testPostgresTarget.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + tables := []sqlname.NameTuple{ + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "foo")}, + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "bar")}, + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "unique_table")}, + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "table1")}, + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "table2")}, + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "non_pk1")}, + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "non_pk2")}, + } + + expectedTables := []sqlname.NameTuple{ + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "foo")}, + {CurrentName: sqlname.NewObjectName(POSTGRESQL, "test_schema", "test_schema", "bar")}, + } + + actualTables := testPostgresTarget.GetNonEmptyTables(tables) + testutils.AssertEqualNameTuplesSlice(t, expectedTables, actualTables) +} diff --git a/yb-voyager/src/tgtdb/yugabytedb_test.go b/yb-voyager/src/tgtdb/yugabytedb_test.go index af2266446..755ed46d6 100644 --- a/yb-voyager/src/tgtdb/yugabytedb_test.go +++ b/yb-voyager/src/tgtdb/yugabytedb_test.go @@ -1,33 +1,40 @@ +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package tgtdb import ( - "context" "database/sql" "fmt" "strings" "testing" + _ "github.com/jackc/pgx/v5/stdlib" + log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" - "github.com/yugabyte/yb-voyager/yb-voyager/src/testutils" - "github.com/yugabyte/yb-voyager/yb-voyager/testcontainers" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) func TestCreateVoyagerSchemaYB(t *testing.T) { - ctx := context.Background() - - // Start a YugabyteDB container - ybContainer, host, port, err := testcontainers.StartDBContainer(ctx, testcontainers.YUGABYTEDB) - assert.NoError(t, err, "Failed to start YugabyteDB container") - defer ybContainer.Terminate(ctx) - - // Connect to the database - dsn := fmt.Sprintf("host=%s port=%s user=yugabyte password=yugabyte dbname=yugabyte sslmode=disable", host, port.Port()) - db, err := sql.Open("postgres", dsn) + db, err := sql.Open("pgx", testYugabyteDBTarget.GetConnectionString()) assert.NoError(t, err) defer db.Close() // Wait for the database to be ready - err = testcontainers.WaitForDBToBeReady(db) + err = testutils.WaitForDBToBeReady(db) assert.NoError(t, err) // Initialize the TargetYugabyteDB instance @@ -80,3 +87,61 @@ func TestCreateVoyagerSchemaYB(t *testing.T) { }) } } + +func TestYugabyteGetNonEmptyTables(t *testing.T) { + testYugabyteDBTarget.ExecuteSqls( + `CREATE SCHEMA test_schema`, + `CREATE TABLE test_schema.foo ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.foo values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.bar ( + id INT PRIMARY KEY, + name VARCHAR + );`, + `INSERT into test_schema.bar values (1, 'abc'), (2, 'xyz');`, + `CREATE TABLE test_schema.unique_table ( + id SERIAL PRIMARY KEY, + email VARCHAR(100), + phone VARCHAR(100), + address VARCHAR(255), + UNIQUE (email, phone) + );`, + `CREATE TABLE test_schema.table1 ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) + );`, + `CREATE TABLE test_schema.table2 ( + id SERIAL PRIMARY KEY, + email VARCHAR(100) + );`, + `CREATE TABLE test_schema.non_pk1( + id INT, + name VARCHAR(255) + );`, + `CREATE TABLE test_schema.non_pk2( + id INT, + name VARCHAR(255) + );`) + defer testYugabyteDBTarget.ExecuteSqls(`DROP SCHEMA test_schema CASCADE;`) + + tables := []sqlname.NameTuple{ + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "foo")}, + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "bar")}, + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "unique_table")}, + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "table1")}, + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "table2")}, + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "non_pk1")}, + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "non_pk2")}, + } + + expectedTables := []sqlname.NameTuple{ + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "foo")}, + {CurrentName: sqlname.NewObjectName(YUGABYTEDB, "test_schema", "test_schema", "bar")}, + } + + actualTables := testYugabyteDBTarget.GetNonEmptyTables(tables) + log.Infof("non empty tables: %+v\n", actualTables) + testutils.AssertEqualNameTuplesSlice(t, expectedTables, actualTables) +} diff --git a/yb-voyager/test/containers/helpers.go b/yb-voyager/test/containers/helpers.go new file mode 100644 index 000000000..cd3afee88 --- /dev/null +++ b/yb-voyager/test/containers/helpers.go @@ -0,0 +1,60 @@ +package testcontainers + +import ( + "context" + _ "embed" + "fmt" + "io" + + log "github.com/sirupsen/logrus" + + "github.com/testcontainers/testcontainers-go" +) + +const ( + DEFAULT_PG_PORT = "5432" + DEFAULT_YB_PORT = "5433" + DEFAULT_ORACLE_PORT = "1521" + DEFAULT_MYSQL_PORT = "3306" + + POSTGRESQL = "postgresql" + YUGABYTEDB = "yugabytedb" + ORACLE = "oracle" + MYSQL = "mysql" +) + +//go:embed test_schemas/postgresql_schema.sql +var postgresInitSchemaFile []byte + +//go:embed test_schemas/oracle_schema.sql +var oracleInitSchemaFile []byte + +//go:embed test_schemas/mysql_schema.sql +var mysqlInitSchemaFile []byte + +//go:embed test_schemas/yugabytedb_schema.sql +var yugabytedbInitSchemaFile []byte + +func printContainerLogs(container testcontainers.Container) { + if container == nil { + log.Printf("Cannot fetch logs: container is nil") + return + } + + containerID := container.GetContainerID() + logs, err := container.Logs(context.Background()) + if err != nil { + log.Printf("Error fetching logs for container %s: %v", containerID, err) + return + } + defer logs.Close() + + // Read the logs + logData, err := io.ReadAll(logs) + if err != nil { + log.Printf("Error reading logs for container %s: %v", containerID, err) + return + } + + fmt.Printf("=== Logs for container %s ===\n%s\n=== End of Logs for container %s ===\n", containerID, string(logData), containerID) +} diff --git a/yb-voyager/test/containers/mysql_container.go b/yb-voyager/test/containers/mysql_container.go new file mode 100644 index 000000000..c2d8930cf --- /dev/null +++ b/yb-voyager/test/containers/mysql_container.go @@ -0,0 +1,149 @@ +package testcontainers + +import ( + "context" + "database/sql" + "fmt" + "os" + "time" + + "github.com/docker/go-connections/nat" + log "github.com/sirupsen/logrus" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" +) + +type MysqlContainer struct { + ContainerConfig + container testcontainers.Container + db *sql.DB +} + +func (ms *MysqlContainer) Start(ctx context.Context) (err error) { + if ms.container != nil { + utils.PrintAndLog("Mysql-%s container already running", ms.DBVersion) + return nil + } + + // since these Start() can be called from anywhere so need a way to ensure that correct files(without needing abs path) are picked from project directories + tmpFile, err := os.CreateTemp(os.TempDir(), "mysql_schema.sql") + if err != nil { + return fmt.Errorf("failed to create temp schema file: %w", err) + } + defer tmpFile.Close() + + if _, err := tmpFile.Write(mysqlInitSchemaFile); err != nil { + return fmt.Errorf("failed to write to temp schema file: %w", err) + } + + req := testcontainers.ContainerRequest{ + // TODO: verify the docker images being used are the correct/certified ones + Image: fmt.Sprintf("mysql:%s", ms.DBVersion), + ExposedPorts: []string{"3306/tcp"}, + Env: map[string]string{ + "MYSQL_ROOT_PASSWORD": ms.Password, + "MYSQL_USER": ms.User, + "MYSQL_PASSWORD": ms.Password, + "MYSQL_DATABASE": ms.DBName, + }, + WaitingFor: wait.ForListeningPort("3306/tcp").WithStartupTimeout(2 * time.Minute).WithPollInterval(5 * time.Second), + Files: []testcontainers.ContainerFile{ + { + HostFilePath: tmpFile.Name(), + ContainerFilePath: "docker-entrypoint-initdb.d/mysql_schema.sql", + FileMode: 0755, + }, + }, + } + + ms.container, err = testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + printContainerLogs(ms.container) + if err != nil { + return err + } + + dsn := ms.GetConnectionString() + db, err := sql.Open("mysql", dsn) + if err != nil { + return fmt.Errorf("failed to open mysql connection: %w", err) + } + + if err = db.Ping(); err != nil { + db.Close() + return fmt.Errorf("failed to ping mysql after connection: %w", err) + } + + // Store the DB connection for reuse + ms.db = db + + return nil +} + +func (ms *MysqlContainer) Terminate(ctx context.Context) { + if ms == nil { + return + } + + // Close the DB connection if it exists + if ms.db != nil { + if err := ms.db.Close(); err != nil { + log.Errorf("failed to close mysql db connection: %v", err) + } + } + + err := ms.container.Terminate(ctx) + if err != nil { + log.Errorf("failed to terminate mysql container: %v", err) + } +} + +func (ms *MysqlContainer) GetHostPort() (string, int, error) { + if ms.container == nil { + return "", -1, fmt.Errorf("mysql container is not started: nil") + } + + ctx := context.Background() + host, err := ms.container.Host(ctx) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch host for mysql container: %w", err) + } + + port, err := ms.container.MappedPort(ctx, nat.Port(DEFAULT_MYSQL_PORT)) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch mapped port for mysql container: %w", err) + } + + return host, port.Int(), nil +} + +func (ms *MysqlContainer) GetConfig() ContainerConfig { + return ms.ContainerConfig +} + +func (ms *MysqlContainer) GetConnectionString() string { + host, port, err := ms.GetHostPort() + if err != nil { + utils.ErrExit("failed to get host port for mysql connection string: %v", err) + } + + // DSN format: user:password@tcp(host:port)/dbname + return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", + ms.User, ms.Password, host, port, ms.DBName) +} + +func (ms *MysqlContainer) ExecuteSqls(sqls ...string) { + if ms.db == nil { + utils.ErrExit("db connection not initialized for mysql container") + } + + for _, sqlStmt := range sqls { + _, err := ms.db.Exec(sqlStmt) + if err != nil { + utils.ErrExit("failed to execute sql '%s': %w", sqlStmt, err) + } + } +} diff --git a/yb-voyager/test/containers/oracle_container.go b/yb-voyager/test/containers/oracle_container.go new file mode 100644 index 000000000..8fb8218de --- /dev/null +++ b/yb-voyager/test/containers/oracle_container.go @@ -0,0 +1,107 @@ +package testcontainers + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/docker/go-connections/nat" + log "github.com/sirupsen/logrus" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" +) + +type OracleContainer struct { + ContainerConfig + container testcontainers.Container +} + +func (ora *OracleContainer) Start(ctx context.Context) (err error) { + if ora.container != nil { + utils.PrintAndLog("Oracle-%s container already running", ora.DBVersion) + return nil + } + + // since these Start() can be called from anywhere so need a way to ensure that correct files(without needing abs path) are picked from project directories + tmpFile, err := os.CreateTemp(os.TempDir(), "oracle_schema.sql") + if err != nil { + return fmt.Errorf("failed to create temp schema file: %w", err) + } + defer tmpFile.Close() + + if _, err := tmpFile.Write(oracleInitSchemaFile); err != nil { + return fmt.Errorf("failed to write to temp schema file: %w", err) + } + + // refer: https://hub.docker.com/r/gvenzl/oracle-xe + req := testcontainers.ContainerRequest{ + // TODO: verify the docker images being used are the correct/certified ones (No license issue) + Image: fmt.Sprintf("gvenzl/oracle-xe:%s", ora.DBVersion), + ExposedPorts: []string{"1521/tcp"}, + Env: map[string]string{ + "ORACLE_PASSWORD": ora.Password, // for SYS user + "ORACLE_DATABASE": ora.DBName, + "APP_USER": ora.User, + "APP_USER_PASSWORD": ora.Password, + }, + WaitingFor: wait.ForLog("DATABASE IS READY TO USE").WithStartupTimeout(2 * time.Minute).WithPollInterval(5 * time.Second), + Files: []testcontainers.ContainerFile{ + { + HostFilePath: tmpFile.Name(), + ContainerFilePath: "docker-entrypoint-initdb.d/oracle_schema.sql", + FileMode: 0755, + }, + }, + } + + ora.container, err = testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + printContainerLogs(ora.container) + return err +} + +func (ora *OracleContainer) Terminate(ctx context.Context) { + if ora == nil { + return + } + + err := ora.container.Terminate(ctx) + if err != nil { + log.Errorf("failed to terminate oracle container: %v", err) + } +} + +func (ora *OracleContainer) GetHostPort() (string, int, error) { + if ora.container == nil { + return "", -1, fmt.Errorf("oracle container is not started: nil") + } + + ctx := context.Background() + host, err := ora.container.Host(ctx) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch host for oracle container: %w", err) + } + + port, err := ora.container.MappedPort(ctx, nat.Port(DEFAULT_ORACLE_PORT)) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch mapped port for oracle container: %w", err) + } + + return host, port.Int(), nil +} + +func (ora *OracleContainer) GetConfig() ContainerConfig { + return ora.ContainerConfig +} + +func (ora *OracleContainer) GetConnectionString() string { + panic("GetConnectionString() not implemented yet for oracle") +} + +func (ora *OracleContainer) ExecuteSqls(sqls ...string) { + +} diff --git a/yb-voyager/test/containers/postgres_container.go b/yb-voyager/test/containers/postgres_container.go new file mode 100644 index 000000000..539763f7d --- /dev/null +++ b/yb-voyager/test/containers/postgres_container.go @@ -0,0 +1,151 @@ +package testcontainers + +import ( + "context" + "database/sql" + "fmt" + "os" + "time" + + "github.com/docker/go-connections/nat" + log "github.com/sirupsen/logrus" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" +) + +type PostgresContainer struct { + ContainerConfig + container testcontainers.Container + db *sql.DB +} + +func (pg *PostgresContainer) Start(ctx context.Context) (err error) { + if pg.container != nil { + utils.PrintAndLog("Postgres-%s container already running", pg.DBVersion) + return nil + } + + // since these Start() can be called from anywhere so need a way to ensure that correct files(without needing abs path) are picked from project directories + tmpFile, err := os.CreateTemp(os.TempDir(), "postgresql_schema.sql") + if err != nil { + return fmt.Errorf("failed to create temp schema file: %w", err) + } + defer tmpFile.Close() + + if _, err := tmpFile.Write(postgresInitSchemaFile); err != nil { + return fmt.Errorf("failed to write to temp schema file: %w", err) + } + + req := testcontainers.ContainerRequest{ + // TODO: verify the docker images being used are the correct/certified ones + Image: fmt.Sprintf("postgres:%s", pg.DBVersion), + ExposedPorts: []string{"5432/tcp"}, + Env: map[string]string{ + "POSTGRES_USER": pg.User, + "POSTGRES_PASSWORD": pg.Password, + "POSTGRES_DB": pg.DBName, // NOTE: PG image makes the database with same name as user if not specific + }, + WaitingFor: wait.ForAll( + wait.ForListeningPort("5432/tcp").WithStartupTimeout(2*time.Minute).WithPollInterval(5*time.Second), + wait.ForLog("database system is ready to accept connections").WithStartupTimeout(3*time.Minute).WithPollInterval(5*time.Second), + ), + Files: []testcontainers.ContainerFile{ + { + HostFilePath: tmpFile.Name(), + ContainerFilePath: "docker-entrypoint-initdb.d/postgresql_schema.sql", + FileMode: 0755, + }, + }, + } + + pg.container, err = testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + + printContainerLogs(pg.container) + if err != nil { + return err + } + + dsn := pg.GetConnectionString() + db, err := sql.Open("pgx", dsn) + if err != nil { + return fmt.Errorf("failed to open postgres connection: %w", err) + } + + if err := db.Ping(); err != nil { + db.Close() + pg.container.Terminate(ctx) + return fmt.Errorf("failed to ping postgres after connection: %w", err) + } + + // Store the DB connection for reuse + pg.db = db + return nil +} + +func (pg *PostgresContainer) Terminate(ctx context.Context) { + if pg == nil { + return + } + + // Close the DB connection if it exists + if pg.db != nil { + if err := pg.db.Close(); err != nil { + log.Errorf("failed to close postgres db connection: %v", err) + } + } + + err := pg.container.Terminate(ctx) + if err != nil { + log.Errorf("failed to terminate postgres container: %v", err) + } +} + +func (pg *PostgresContainer) GetHostPort() (string, int, error) { + if pg.container == nil { + return "", -1, fmt.Errorf("postgres container is not started: nil") + } + + ctx := context.Background() + host, err := pg.container.Host(ctx) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch host for postgres container: %w", err) + } + + port, err := pg.container.MappedPort(ctx, nat.Port(DEFAULT_PG_PORT)) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch mapped port for postgres container: %w", err) + } + + return host, port.Int(), nil +} + +func (pg *PostgresContainer) GetConfig() ContainerConfig { + return pg.ContainerConfig +} + +func (pg *PostgresContainer) GetConnectionString() string { + config := pg.GetConfig() + host, port, err := pg.GetHostPort() + if err != nil { + utils.ErrExit("failed to get host port for postgres connection string: %v", err) + } + + return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s", config.User, config.Password, host, port, config.DBName) +} + +func (pg *PostgresContainer) ExecuteSqls(sqls ...string) { + if pg.db == nil { + utils.ErrExit("db connection not initialized for postgres container") + } + + for _, sqlStmt := range sqls { + _, err := pg.db.Exec(sqlStmt) + if err != nil { + utils.ErrExit("failed to execute sql '%s': %w", sqlStmt, err) + } + } +} diff --git a/yb-voyager/test/containers/test_schemas/mysql_schema.sql b/yb-voyager/test/containers/test_schemas/mysql_schema.sql new file mode 100644 index 000000000..e72f4bcb1 --- /dev/null +++ b/yb-voyager/test/containers/test_schemas/mysql_schema.sql @@ -0,0 +1,6 @@ +-- TODO: create user as per User creation steps in docs and use that in tests + +-- Grant CREATE, ALTER, DROP privileges globally to 'ybvoyager' +GRANT CREATE, ALTER, DROP ON *.* TO 'ybvoyager'@'%' WITH GRANT OPTION; +-- Apply the changes +FLUSH PRIVILEGES; \ No newline at end of file diff --git a/yb-voyager/test/containers/test_schemas/oracle_schema.sql b/yb-voyager/test/containers/test_schemas/oracle_schema.sql new file mode 100644 index 000000000..95b3a3a9b --- /dev/null +++ b/yb-voyager/test/containers/test_schemas/oracle_schema.sql @@ -0,0 +1,47 @@ +-- TODO: create user as per User creation steps in docs and use that in tests + +-- Used ORACLE_DATABASE=DMS i.e. pluggable database to create APP_USER +ALTER SESSION SET CONTAINER = "DMS"; + + +-- creating tables under YBVOYAGER schema, same as APP_USER +CREATE TABLE YBVOYAGER.foo ( + id NUMBER PRIMARY KEY, + name VARCHAR2(255) +); + +CREATE TABLE YBVOYAGER.bar ( + id NUMBER PRIMARY KEY, + name VARCHAR2(255) +); + +CREATE TABLE YBVOYAGER.unique_table ( + id NUMBER PRIMARY KEY, + email VARCHAR2(100), + phone VARCHAR2(100), + address VARCHAR2(255), + CONSTRAINT email_phone_unq UNIQUE (email, phone) +); + +CREATE UNIQUE INDEX YBVOYAGER.unique_address_idx ON YBVOYAGER.unique_table (address); + +CREATE TABLE YBVOYAGER.table1 ( + id NUMBER PRIMARY KEY, + name VARCHAR2(100) +); + +CREATE TABLE YBVOYAGER.table2 ( + id NUMBER PRIMARY KEY, + email VARCHAR2(100) +); + + +CREATE TABLE YBVOYAGER.non_pk1 ( + id NUMBER, + name VARCHAR2(10) +); + +CREATE TABLE YBVOYAGER.non_pk2 ( + id NUMBER, + name VARCHAR2(10) +); \ No newline at end of file diff --git a/yb-voyager/test/containers/test_schemas/postgresql_schema.sql b/yb-voyager/test/containers/test_schemas/postgresql_schema.sql new file mode 100644 index 000000000..36bda657a --- /dev/null +++ b/yb-voyager/test/containers/test_schemas/postgresql_schema.sql @@ -0,0 +1 @@ +-- TODO: create source migration user as per User creation steps in docs and use that in tests diff --git a/yb-voyager/test/containers/test_schemas/yugabytedb_schema.sql b/yb-voyager/test/containers/test_schemas/yugabytedb_schema.sql new file mode 100644 index 000000000..c36ddc5b9 --- /dev/null +++ b/yb-voyager/test/containers/test_schemas/yugabytedb_schema.sql @@ -0,0 +1 @@ +-- TODO: create user as per User creation steps in docs and use that in tests diff --git a/yb-voyager/test/containers/testcontainers.go b/yb-voyager/test/containers/testcontainers.go new file mode 100644 index 000000000..c9fce4b15 --- /dev/null +++ b/yb-voyager/test/containers/testcontainers.go @@ -0,0 +1,143 @@ +package testcontainers + +import ( + "context" + "fmt" + "sync" + + "github.com/samber/lo" + log "github.com/sirupsen/logrus" +) + +// containerRegistry to ensure one container per database(dbtype+version) [Singleton Pattern] +// Limitation - go test spawns different process for running tests of each package, hence the containers won't be shared across packages. +var ( + containerRegistry = make(map[string]TestContainer) + registryMutex sync.Mutex +) + +type TestContainer interface { + Start(ctx context.Context) error + Terminate(ctx context.Context) + GetHostPort() (string, int, error) + GetConfig() ContainerConfig + GetConnectionString() string + /* + TODOs + // Function to run sql script for a specific test case + SetupSqlScript(scriptName string, dbName string) error + + // Add Capability to run multiple versions of a dbtype parallely + */ + ExecuteSqls(sqls ...string) +} + +type ContainerConfig struct { + DBVersion string + User string + Password string + DBName string + Schema string +} + +func NewTestContainer(dbType string, containerConfig *ContainerConfig) TestContainer { + registryMutex.Lock() + defer registryMutex.Unlock() + + // initialise containerConfig struct if nothing is provided + if containerConfig == nil { + containerConfig = &ContainerConfig{} + } + setContainerConfigDefaultsIfNotProvided(dbType, containerConfig) + + // check if container is already created after fetching default configs + containerName := fmt.Sprintf("%s-%s", dbType, containerConfig.DBVersion) + if container, exists := containerRegistry[containerName]; exists { + log.Infof("container '%s' already exists in the registry", containerName) + return container + } + + var testContainer TestContainer + switch dbType { + case POSTGRESQL: + testContainer = &PostgresContainer{ + ContainerConfig: *containerConfig, + } + case YUGABYTEDB: + testContainer = &YugabyteDBContainer{ + ContainerConfig: *containerConfig, + } + case ORACLE: + testContainer = &OracleContainer{ + ContainerConfig: *containerConfig, + } + case MYSQL: + testContainer = &MysqlContainer{ + ContainerConfig: *containerConfig, + } + default: + panic(fmt.Sprintf("unsupported db type '%q' for creating test container\n", dbType)) + } + + containerRegistry[containerName] = testContainer + return testContainer +} + +/* +Challenges in golang for running this a teardown step +1. In golang when you execute go test in the top level folder it executes all the tests one by one. +2. Where each defined package, can have its TestMain() which can control the setup and teardown steps for that package +3. There is no way to run these before/after the tests of first/last package in codebase + +Potential solution: Implement a counter(total=number_of_package) based logic to execute teardown(i.e. TerminateAllContainers() in our case) +Figure out the best solution. + +For now we can rely on TestContainer ryuk(the container repear), which terminates all the containers after the process exits. +But the test framework should have capability of terminating all containers at the end. +*/ +func TerminateAllContainers() { + registryMutex.Lock() + defer registryMutex.Unlock() + + ctx := context.Background() + for name, container := range containerRegistry { + log.Infof("terminating the container '%s'", name) + container.Terminate(ctx) + } +} + +func setContainerConfigDefaultsIfNotProvided(dbType string, config *ContainerConfig) { + // TODO: discuss and decide the default DBVersion values for each dbtype + + switch dbType { + case POSTGRESQL: + config.User = lo.Ternary(config.User == "", "ybvoyager", config.User) + config.Password = lo.Ternary(config.Password == "", "passsword", config.Password) + config.DBVersion = lo.Ternary(config.DBVersion == "", "11", config.DBVersion) + config.Schema = lo.Ternary(config.Schema == "", "public", config.Schema) + config.DBName = lo.Ternary(config.DBName == "", "postgres", config.DBName) + + case YUGABYTEDB: + config.User = lo.Ternary(config.User == "", "yugabyte", config.User) // ybdb docker doesn't create specified user + config.Password = lo.Ternary(config.Password == "", "passsword", config.Password) + config.DBVersion = lo.Ternary(config.DBVersion == "", "2.20.7.1-b10", config.DBVersion) + config.Schema = lo.Ternary(config.Schema == "", "public", config.Schema) + config.DBName = lo.Ternary(config.DBName == "", "yugabyte", config.DBName) + + case ORACLE: + config.User = lo.Ternary(config.User == "", "ybvoyager", config.User) + config.Password = lo.Ternary(config.Password == "", "passsword", config.Password) + config.DBVersion = lo.Ternary(config.DBVersion == "", "21", config.DBVersion) + config.Schema = lo.Ternary(config.Schema == "", "YBVOYAGER", config.Schema) + config.DBName = lo.Ternary(config.DBName == "", "DMS", config.DBName) + + case MYSQL: + config.User = lo.Ternary(config.User == "", "ybvoyager", config.User) + config.Password = lo.Ternary(config.Password == "", "passsword", config.Password) + config.DBVersion = lo.Ternary(config.DBVersion == "", "8.4", config.DBVersion) + config.DBName = lo.Ternary(config.DBName == "", "dms", config.DBName) + + default: + panic(fmt.Sprintf("unsupported db type '%q' for creating test container\n", dbType)) + } +} diff --git a/yb-voyager/test/containers/yugabytedb_container.go b/yb-voyager/test/containers/yugabytedb_container.go new file mode 100644 index 000000000..822a7f694 --- /dev/null +++ b/yb-voyager/test/containers/yugabytedb_container.go @@ -0,0 +1,129 @@ +package testcontainers + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/docker/go-connections/nat" + "github.com/jackc/pgx/v5" + log "github.com/sirupsen/logrus" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" +) + +type YugabyteDBContainer struct { + ContainerConfig + container testcontainers.Container +} + +func (yb *YugabyteDBContainer) Start(ctx context.Context) (err error) { + if yb.container != nil { + utils.PrintAndLog("YugabyteDB-%s container already running", yb.DBVersion) + return nil + } + + // since these Start() can be called from anywhere so need a way to ensure that correct files(without needing abs path) are picked from project directories + tmpFile, err := os.CreateTemp(os.TempDir(), "yugabytedb_schema.sql") + if err != nil { + return fmt.Errorf("failed to create temp schema file: %w", err) + } + defer tmpFile.Close() + + if _, err := tmpFile.Write(yugabytedbInitSchemaFile); err != nil { + return fmt.Errorf("failed to write to temp schema file: %w", err) + } + + // this will create a 1 Node RF-1 cluster + req := testcontainers.ContainerRequest{ + Image: fmt.Sprintf("yugabytedb/yugabyte:%s", yb.DBVersion), + ExposedPorts: []string{"5433/tcp", "15433/tcp", "7000/tcp", "9000/tcp", "9042/tcp"}, + Cmd: []string{ + "bin/yugabyted", + "start", + "--daemon=false", + "--ui=false", + "--initial_scripts_dir=/home/yugabyte/initial-scripts", + }, + WaitingFor: wait.ForAll( + wait.ForListeningPort("5433/tcp").WithStartupTimeout(2*time.Minute).WithPollInterval(5*time.Second), + wait.ForLog("Data placement constraint successfully verified").WithStartupTimeout(3*time.Minute).WithPollInterval(1*time.Second), + ), + Files: []testcontainers.ContainerFile{ + { + HostFilePath: tmpFile.Name(), + ContainerFilePath: "/home/yugabyte/initial-scripts/yugabytedb_schema.sql", + FileMode: 0755, + }, + }, + } + + yb.container, err = testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + printContainerLogs(yb.container) + return err +} + +func (yb *YugabyteDBContainer) Terminate(ctx context.Context) { + if yb == nil { + return + } + + err := yb.container.Terminate(ctx) + if err != nil { + log.Errorf("failed to terminate yugabytedb container: %v", err) + } +} + +func (yb *YugabyteDBContainer) GetHostPort() (string, int, error) { + if yb.container == nil { + return "", -1, fmt.Errorf("yugabytedb container is not started: nil") + } + + ctx := context.Background() + host, err := yb.container.Host(ctx) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch host for yugabytedb container: %w", err) + } + + port, err := yb.container.MappedPort(ctx, nat.Port(DEFAULT_YB_PORT)) + if err != nil { + return "", -1, fmt.Errorf("failed to fetch mapped port for yugabytedb container: %w", err) + } + + return host, port.Int(), nil +} + +func (yb *YugabyteDBContainer) GetConfig() ContainerConfig { + return yb.ContainerConfig +} + +func (yb *YugabyteDBContainer) GetConnectionString() string { + config := yb.GetConfig() + host, port, err := yb.GetHostPort() + if err != nil { + utils.ErrExit("failed to get host port for yugabytedb connection string: %v", err) + } + + return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s", config.User, config.Password, host, port, config.DBName) +} + +func (yb *YugabyteDBContainer) ExecuteSqls(sqls ...string) { + connStr := yb.GetConnectionString() + conn, err := pgx.Connect(context.Background(), connStr) + if err != nil { + utils.ErrExit("failed to connect postgres for executing sqls: %w", err) + } + defer conn.Close(context.Background()) + + for _, sql := range sqls { + _, err := conn.Exec(context.Background(), sql) + if err != nil { + utils.ErrExit("failed to execute sql '%s': %w", sql, err) + } + } +} diff --git a/yb-voyager/src/testutils/testing_utils.go b/yb-voyager/test/utils/testutils.go similarity index 89% rename from yb-voyager/src/testutils/testing_utils.go rename to yb-voyager/test/utils/testutils.go index 2a10007de..74c30bc44 100644 --- a/yb-voyager/src/testutils/testing_utils.go +++ b/yb-voyager/test/utils/testutils.go @@ -6,11 +6,14 @@ import ( "os" "reflect" "regexp" + "sort" "strings" "testing" + "time" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/sqlname" ) type ColumnPropertiesSqlite struct { @@ -201,10 +204,10 @@ func CheckTableStructurePG(t *testing.T, db *sql.DB, schema, table string, expec func checkPrimaryKeyOfTablePG(t *testing.T, db *sql.DB, schema, table string, expectedColumns map[string]ColumnPropertiesPG) { // Validate primary keys queryPrimaryKeys := ` - SELECT conrelid::regclass AS table_name, - conname AS primary_key, + SELECT conrelid::regclass AS table_name, + conname AS primary_key, pg_get_constraintdef(oid) - FROM pg_constraint + FROM pg_constraint WHERE contype = 'p' -- 'p' indicates primary key AND conrelid::regclass::text = $1 ORDER BY conrelid::regclass::text, contype DESC;` @@ -350,3 +353,50 @@ func CheckTableExistencePG(t *testing.T, db *sql.DB, schema string, expectedTabl } } } + +// === assertion helper functions +func AssertEqualStringSlices(t *testing.T, expected, actual []string) { + t.Helper() + if len(expected) != len(actual) { + t.Errorf("Mismatch in slice length. Expected: %v, Actual: %v", expected, actual) + } + + sort.Strings(expected) + sort.Strings(actual) + assert.Equal(t, expected, actual) +} + +func AssertEqualSourceNameSlices(t *testing.T, expected, actual []*sqlname.SourceName) { + SortSourceNames(expected) + SortSourceNames(actual) + assert.Equal(t, expected, actual) +} + +func SortSourceNames(tables []*sqlname.SourceName) { + sort.Slice(tables, func(i, j int) bool { + return tables[i].Qualified.MinQuoted < tables[j].Qualified.MinQuoted + }) +} + +func AssertEqualNameTuplesSlice(t *testing.T, expected, actual []sqlname.NameTuple) { + sortNameTuples(expected) + sortNameTuples(actual) + assert.Equal(t, expected, actual) +} + +func sortNameTuples(tables []sqlname.NameTuple) { + sort.Slice(tables, func(i, j int) bool { + return tables[i].ForOutput() < tables[j].ForOutput() + }) +} + +// waitForDBConnection waits until the database is ready for connections. +func WaitForDBToBeReady(db *sql.DB) error { + for i := 0; i < 12; i++ { + if err := db.Ping(); err == nil { + return nil + } + time.Sleep(5 * time.Second) + } + return fmt.Errorf("database did not become ready in time") +} diff --git a/yb-voyager/testcontainers/testcontainers.go b/yb-voyager/testcontainers/testcontainers.go deleted file mode 100644 index 336d7b2e3..000000000 --- a/yb-voyager/testcontainers/testcontainers.go +++ /dev/null @@ -1,108 +0,0 @@ -package testcontainers - -import ( - "context" - "database/sql" - "fmt" - "time" - - "github.com/docker/go-connections/nat" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -const ( - POSTGRESQL = "postgresql" - YUGABYTEDB = "yugabytedb" -) - -func StartDBContainer(ctx context.Context, dbType string) (testcontainers.Container, string, nat.Port, error) { - switch dbType { - case POSTGRESQL: - return startPostgresContainer(ctx) - case YUGABYTEDB: - return startYugabyteDBContainer(ctx) - default: - return nil, "", "", fmt.Errorf("unsupported database type: %s", dbType) - } -} - -func startPostgresContainer(ctx context.Context) (container testcontainers.Container, host string, port nat.Port, err error) { - // Create a PostgreSQL TestContainer - req := testcontainers.ContainerRequest{ - Image: "postgres:latest", // Use the latest PostgreSQL image - ExposedPorts: []string{"5432/tcp"}, - Env: map[string]string{ - "POSTGRES_USER": "testuser", // Set PostgreSQL username - "POSTGRES_PASSWORD": "testpassword", // Set PostgreSQL password - "POSTGRES_DB": "testdb", // Set PostgreSQL database name - }, - WaitingFor: wait.ForListeningPort("5432/tcp").WithStartupTimeout(30 * 1e9), // Wait for PostgreSQL to be ready - } - - // Start the container - pgContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - - if err != nil { - return nil, "", "", err - } - - // Get the container's host and port - host, err = pgContainer.Host(ctx) - if err != nil { - return nil, "", "", err - } - - port, err = pgContainer.MappedPort(ctx, "5432") - if err != nil { - return nil, "", "", err - } - - return pgContainer, host, port, nil -} - -func startYugabyteDBContainer(ctx context.Context) (container testcontainers.Container, host string, port nat.Port, err error) { - // Create a YugabyteDB TestContainer - req := testcontainers.ContainerRequest{ - Image: "yugabytedb/yugabyte:latest", - ExposedPorts: []string{"5433/tcp"}, - WaitingFor: wait.ForListeningPort("5433/tcp"), - Cmd: []string{"bin/yugabyted", "start", "--daemon=false"}, - } - - // Start the container - ybContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - if err != nil { - return nil, "", "", err - } - - // Get the container's host and port - host, err = ybContainer.Host(ctx) - if err != nil { - return nil, "", "", err - } - - port, err = ybContainer.MappedPort(ctx, "5433") - if err != nil { - return nil, "", "", err - } - - return ybContainer, host, port, nil -} - -// waitForDBConnection waits until the database is ready for connections. -func WaitForDBToBeReady(db *sql.DB) error { - for i := 0; i < 12; i++ { - if err := db.Ping(); err == nil { - return nil - } - time.Sleep(5 * time.Second) - } - return fmt.Errorf("database did not become ready in time") -} From f19b943d9e08929dce609fab1ecd8b39cb90fc57 Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Thu, 26 Dec 2024 16:08:42 +0530 Subject: [PATCH 10/17] Added a new detector for catching unsupported COPY command structures in PG (#2106) --- .../expectedAssessmentReport.json | 35 +++++++-- .../unsupported_query_constructs.sql | 30 +++++++- yb-voyager/src/query/queryissue/constants.go | 2 + yb-voyager/src/query/queryissue/detectors.go | 77 +++++++++++++++---- yb-voyager/src/query/queryissue/issues_dml.go | 26 +++++++ .../src/query/queryissue/issues_dml_test.go | 28 +++++++ .../query/queryissue/parser_issue_detector.go | 3 +- .../queryissue/parser_issue_detector_test.go | 39 ++++++++++ .../src/query/queryparser/helpers_protomsg.go | 10 ++- .../src/query/queryparser/traversal_proto.go | 1 + 10 files changed, 227 insertions(+), 24 deletions(-) diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 1057ad9fd..5a861d641 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -38,15 +38,15 @@ }, { "ObjectType": "SEQUENCE", - "TotalCount": 28, + "TotalCount": 29, "InvalidCount": 0, - "ObjectNames": "public.ordersentry_order_id_seq, public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.employees_employee_id_seq, public.employees2_id_seq, public.ext_test_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.orders2_id_seq, public.parent_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" + "ObjectNames": "public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.employees2_id_seq, public.employees3_id_seq, public.employees_employee_id_seq, public.ext_test_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" }, { "ObjectType": "TABLE", - "TotalCount": 67, + "TotalCount": 68, "InvalidCount": 23, - "ObjectNames": "public.ordersentry, public.library_nested, public.orders_lateral, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.sales_region, public.boston, public.c, public.parent_table, public.child_table, public.citext_type, public.combined_tbl, public.documents, public.employees2, public.ext_test, public.foo, public.inet_type, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.orders, public.orders2, public.products, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.sales_region, schema2.boston, schema2.c, schema2.parent_table, schema2.child_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.orders, schema2.orders2, schema2.products, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2, public.employees" + "ObjectNames": "public.employees3, public.ordersentry, public.library_nested, public.orders_lateral, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.sales_region, public.boston, public.c, public.parent_table, public.child_table, public.citext_type, public.combined_tbl, public.documents, public.employees2, public.ext_test, public.foo, public.inet_type, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.orders, public.orders2, public.products, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.sales_region, schema2.boston, schema2.c, schema2.parent_table, schema2.child_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.orders, schema2.orders2, schema2.products, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2, public.employees" }, { "ObjectType": "INDEX", @@ -170,9 +170,10 @@ "test_views.view_table1", "public.library_nested", "public.orders_lateral", - "public.employees" + "public.employees", + "public.employees3" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 73 objects (65 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 74 objects (66 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": [ "public.combined_tbl", "public.citext_type", @@ -1983,6 +1984,20 @@ "ObjectType": "", "ParentTableName": "schema2.mixed_data_types_table1", "SizeInBytes": 8192 + }, + { + "SchemaName": "public", + "ObjectName": "employees3", + "RowCount": 2, + "ColumnCount": 3, + "Reads": 0, + "Writes": 2, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 8192 } ], "Notes": [ @@ -2182,6 +2197,12 @@ "Query": "SELECT lo_create($1)", "DocsLink": "", "MinimumVersionsFixedIn": null + }, + { + "ConstructTypeName": "COPY FROM ... WHERE", + "Query": "COPY employees3 (id, name, age)\nFROM STDIN WITH (FORMAT csv)\nWHERE age \u003e 30", + "DocsLink": "", + "MinimumVersionsFixedIn": null } ], "UnsupportedPlPgSqlObjects": [ @@ -2231,4 +2252,4 @@ "MinimumVersionsFixedIn": null } ] -} \ No newline at end of file +} diff --git a/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql b/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql index cc7251039..284c8f7d0 100644 --- a/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql +++ b/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql @@ -138,4 +138,32 @@ FROM ) AS items; -SELECT lo_create('32142'); \ No newline at end of file +SELECT lo_create('32142'); + +-- Unsupported COPY constructs + +CREATE TABLE IF NOT EXISTS employees3 ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + age INT NOT NULL +); + + +-- COPY FROM with WHERE clause +COPY employees3 (id, name, age) +FROM STDIN WITH (FORMAT csv) +WHERE age > 30; +1,John Smith,25 +2,Jane Doe,34 +3,Bob Johnson,31 +\. + +-- This can be uncommented when we start using PG 17 or later in the tests +-- -- COPY with ON_ERROR clause +-- COPY employees (id, name, age) +-- FROM STDIN WITH (FORMAT csv, ON_ERROR IGNORE ); +-- 4,Adam Smith,22 +-- 5,John Doe,34 +-- 6,Ron Johnson,31 +-- \. + diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index f0f86239a..8d319777e 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -70,6 +70,8 @@ const ( XML_FUNCTIONS_NAME = "XML Functions" REGEX_FUNCTIONS = "REGEX_FUNCTIONS" + COPY_FROM_WHERE = "COPY FROM ... WHERE" + COPY_ON_ERROR = "COPY ... ON_ERROR" ) // Object types diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index 8a57c581d..e6ec4c4cc 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -16,6 +16,8 @@ limitations under the License. package queryissue import ( + "slices" + mapset "github.com/deckarep/golang-set/v2" log "github.com/sirupsen/logrus" "google.golang.org/protobuf/reflect/protoreflect" @@ -204,6 +206,55 @@ func (d *RangeTableFuncDetector) GetIssues() []QueryIssue { return issues } +type CopyCommandUnsupportedConstructsDetector struct { + query string + copyFromWhereConstructDetected bool + copyOnErrorConstructDetected bool +} + +func NewCopyCommandUnsupportedConstructsDetector(query string) *CopyCommandUnsupportedConstructsDetector { + return &CopyCommandUnsupportedConstructsDetector{ + query: query, + } +} + +// Detect if COPY command uses unsupported syntax i.e. COPY FROM ... WHERE and COPY... ON_ERROR +func (d *CopyCommandUnsupportedConstructsDetector) Detect(msg protoreflect.Message) error { + // Check if the message is a COPY statement + if msg.Descriptor().FullName() != queryparser.PG_QUERY_COPYSTSMT_NODE { + return nil // Not a COPY statement, nothing to detect + } + + // Check for COPY FROM ... WHERE clause + fromField := queryparser.GetBoolField(msg, "is_from") + whereField := queryparser.GetMessageField(msg, "where_clause") + if fromField && whereField != nil { + d.copyFromWhereConstructDetected = true + } + + // Check for COPY ... ON_ERROR clause + defNames, err := queryparser.TraverseAndExtractDefNamesFromDefElem(msg) + if err != nil { + log.Errorf("error extracting defnames from COPY statement: %v", err) + } + if slices.Contains(defNames, "on_error") { + d.copyOnErrorConstructDetected = true + } + + return nil +} + +func (d *CopyCommandUnsupportedConstructsDetector) GetIssues() []QueryIssue { + var issues []QueryIssue + if d.copyFromWhereConstructDetected { + issues = append(issues, NewCopyFromWhereIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) + } + if d.copyOnErrorConstructDetected { + issues = append(issues, NewCopyOnErrorIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) + } + return issues +} + type JsonConstructorFuncDetector struct { query string unsupportedJsonConstructorFunctionsDetected mapset.Set[string] @@ -253,19 +304,19 @@ func NewJsonQueryFunctionDetector(query string) *JsonQueryFunctionDetector { func (j *JsonQueryFunctionDetector) Detect(msg protoreflect.Message) error { if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_JSON_TABLE_NODE { /* - SELECT * FROM json_table( - '[{"a":10,"b":20},{"a":30,"b":40}]'::jsonb, - '$[*]' - COLUMNS ( - column_a int4 path '$.a', - column_b int4 path '$.b' - ) - ); - stmts:{stmt:{select_stmt:{target_list:{res_target:{val:{column_ref:{fields:{a_star:{}} location:530}} location:530}} - from_clause:{json_table:{context_item:{raw_expr:{type_cast:{arg:{a_const:{sval:{sval:"[{\"a\":10,\"b\":20},{\"a\":30,\"b\":40}]"} - location:553}} type_name:{names:{string:{sval:"jsonb"}} ..... name_location:-1 location:601} - columns:{json_table_column:{coltype:JTC_REGULAR name:"column_a" type_name:{names:{string:{sval:"int4"}} typemod:-1 location:639} - pathspec:{string:{a_const:{sval:{sval:"$.a"} location:649}} name_location:-1 location:649} ... + SELECT * FROM json_table( + '[{"a":10,"b":20},{"a":30,"b":40}]'::jsonb, + '$[*]' + COLUMNS ( + column_a int4 path '$.a', + column_b int4 path '$.b' + ) + ); + stmts:{stmt:{select_stmt:{target_list:{res_target:{val:{column_ref:{fields:{a_star:{}} location:530}} location:530}} + from_clause:{json_table:{context_item:{raw_expr:{type_cast:{arg:{a_const:{sval:{sval:"[{\"a\":10,\"b\":20},{\"a\":30,\"b\":40}]"} + location:553}} type_name:{names:{string:{sval:"jsonb"}} ..... name_location:-1 location:601} + columns:{json_table_column:{coltype:JTC_REGULAR name:"column_a" type_name:{names:{string:{sval:"int4"}} typemod:-1 location:639} + pathspec:{string:{a_const:{sval:{sval:"$.a"} location:649}} name_location:-1 location:649} ... */ j.unsupportedJsonQueryFunctionsDetected.Add(JSON_TABLE) return nil diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index 43dc8ab07..e6915177c 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -141,3 +141,29 @@ func NewLOFuntionsIssue(objectType string, objectName string, sqlStatement strin } return newQueryIssue(loFunctionsIssue, objectType, objectName, sqlStatement, details) } + +var copyFromWhereIssue = issue.Issue{ + Type: COPY_FROM_WHERE, + TypeName: "COPY FROM ... WHERE", + TypeDescription: "", + Suggestion: "", + GH: "", + DocsLink: "", +} + +func NewCopyFromWhereIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(copyFromWhereIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} + +var copyOnErrorIssue = issue.Issue{ + Type: COPY_ON_ERROR, + TypeName: "COPY ... ON_ERROR", + TypeDescription: "", + Suggestion: "", + GH: "", + DocsLink: "", +} + +func NewCopyOnErrorIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(copyOnErrorIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} diff --git a/yb-voyager/src/query/queryissue/issues_dml_test.go b/yb-voyager/src/query/queryissue/issues_dml_test.go index 5b0cd0151..d24d4f194 100644 --- a/yb-voyager/src/query/queryissue/issues_dml_test.go +++ b/yb-voyager/src/query/queryissue/issues_dml_test.go @@ -58,6 +58,29 @@ func testRegexFunctionsIssue(t *testing.T) { } } +func testCopyOnErrorIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + // In case the COPY ... ON_ERROR construct gets supported in the future, this test will fail with a different error message-something related to the data.csv file not being found. + _, err = conn.Exec(ctx, `COPY pg_largeobject (loid, pageno, data) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE);`) + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "ERROR: option \"on_error\" not recognized (SQLSTATE 42601)", copyOnErrorIssue) + +} + +func testCopyFromWhereIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + // In case the COPY FROM ... WHERE construct gets supported in the future, this test will fail with a different error message-something related to the data.csv file not being found. + _, err = conn.Exec(ctx, `COPY pg_largeobject (loid, pageno, data) FROM '/path/to/data.csv' WHERE loid = 1 WITH (FORMAT csv, HEADER true);`) + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "ERROR: syntax error at or near \"WHERE\" (SQLSTATE 42601)", copyFromWhereIssue) +} + func testJsonConstructorFunctions(t *testing.T) { ctx := context.Background() conn, err := getConn() @@ -145,6 +168,11 @@ func TestDMLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "regex functions", ybVersion), testRegexFunctionsIssue) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "copy on error", ybVersion), testCopyOnErrorIssue) + assert.True(t, success) + + success = t.Run(fmt.Sprintf("%s-%s", "copy from where", ybVersion), testCopyFromWhereIssue) + assert.True(t, success) success = t.Run(fmt.Sprintf("%s-%s", "json constructor functions", ybVersion), testJsonConstructorFunctions) assert.True(t, success) diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector.go b/yb-voyager/src/query/queryissue/parser_issue_detector.go index 8b5f028b9..9c32fca53 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector.go @@ -375,6 +375,7 @@ func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) NewColumnRefDetector(query), NewXmlExprDetector(query), NewRangeTableFuncDetector(query), + NewCopyCommandUnsupportedConstructsDetector(query), NewJsonConstructorFuncDetector(query), NewJsonQueryFunctionDetector(query), } @@ -414,7 +415,7 @@ func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) xmlIssueAdded = true } } - result = append(result, issues...) + result = append(result, issue) } } diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index 1450f4edd..0d6e9d23f 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -649,3 +649,42 @@ func TestRegexFunctionsIssue(t *testing.T) { } } + +func TestCopyUnsupportedConstructIssuesDetected(t *testing.T) { + expectedIssues := map[string][]QueryIssue{ + `COPY my_table FROM '/path/to/data.csv' WHERE col1 > 100;`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY my_table FROM '/path/to/data.csv' WHERE col1 > 100;`)}, + `COPY my_table(col1, col2) FROM '/path/to/data.csv' WHERE col2 = 'test';`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY my_table(col1, col2) FROM '/path/to/data.csv' WHERE col2 = 'test';`)}, + `COPY my_table FROM '/path/to/data.csv' WHERE TRUE;`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY my_table FROM '/path/to/data.csv' WHERE TRUE;`)}, + `COPY employees (id, name, age) + FROM STDIN WITH (FORMAT csv) + WHERE age > 30;`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY employees (id, name, age) + FROM STDIN WITH (FORMAT csv) + WHERE age > 30;`)}, + + `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE);`: {NewCopyOnErrorIssue("DML_QUERY", "", `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE);`)}, + `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP);`: {NewCopyOnErrorIssue("DML_QUERY", "", `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP);`)}, + + `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE) WHERE age > 18;`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE) WHERE age > 18;`), NewCopyOnErrorIssue("DML_QUERY", "", `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE) WHERE age > 18;`)}, + `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP) WHERE name = 'Alice';`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP) WHERE name = 'Alice';`), NewCopyOnErrorIssue("DML_QUERY", "", `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP) WHERE name = 'Alice';`)}, + + `COPY my_table FROM '/path/to/data.csv' WITH (FORMAT csv);`: {}, + `COPY my_table FROM '/path/to/data.csv' WITH (FORMAT text);`: {}, + `COPY my_table FROM '/path/to/data.csv';`: {}, + `COPY my_table FROM '/path/to/data.csv' WITH (DELIMITER ',');`: {}, + `COPY my_table(col1, col2) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true);`: {}, + } + + parserIssueDetector := NewParserIssueDetector() + + for stmt, expectedIssues := range expectedIssues { + issues, err := parserIssueDetector.getDMLIssues(stmt) + fatalIfError(t, err) + assert.Equal(t, len(expectedIssues), len(issues)) + for _, expectedIssue := range expectedIssues { + found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { + return cmp.Equal(expectedIssue, queryIssue) + }) + assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) + } + } +} diff --git a/yb-voyager/src/query/queryparser/helpers_protomsg.go b/yb-voyager/src/query/queryparser/helpers_protomsg.go index e0de00604..8fe62bc82 100644 --- a/yb-voyager/src/query/queryparser/helpers_protomsg.go +++ b/yb-voyager/src/query/queryparser/helpers_protomsg.go @@ -366,6 +366,14 @@ func GetMessageField(msg protoreflect.Message, fieldName string) protoreflect.Me return nil } +func GetBoolField(msg protoreflect.Message, fieldName string) bool { + field := msg.Descriptor().Fields().ByName(protoreflect.Name(fieldName)) + if field != nil && msg.Has(field) { + return msg.Get(field).Bool() + } + return false +} + // GetListField retrieves a list field from a message. func GetListField(msg protoreflect.Message, fieldName string) protoreflect.List { field := msg.Descriptor().Fields().ByName(protoreflect.Name(fieldName)) @@ -403,12 +411,10 @@ func GetSchemaAndObjectName(nameList protoreflect.List) (string, string) { Example: options:{def_elem:{defname:"security_invoker" arg:{string:{sval:"true"}} defaction:DEFELEM_UNSPEC location:32}} options:{def_elem:{defname:"security_barrier" arg:{string:{sval:"false"}} defaction:DEFELEM_UNSPEC location:57}} - Extract all defnames from the def_eleme node */ func TraverseAndExtractDefNamesFromDefElem(msg protoreflect.Message) ([]string, error) { var defNames []string - collectorFunc := func(msg protoreflect.Message) error { if GetMsgFullName(msg) != PG_QUERY_DEFELEM_NODE { return nil diff --git a/yb-voyager/src/query/queryparser/traversal_proto.go b/yb-voyager/src/query/queryparser/traversal_proto.go index c988ee48e..e742a8172 100644 --- a/yb-voyager/src/query/queryparser/traversal_proto.go +++ b/yb-voyager/src/query/queryparser/traversal_proto.go @@ -49,6 +49,7 @@ const ( PG_QUERY_JSON_OBJECT_CONSTRUCTOR_NODE = "pg_query.JsonObjectConstructor" PG_QUERY_JSON_TABLE_NODE = "pg_query.JsonTable" PG_QUERY_VIEWSTMT_NODE = "pg_query.ViewStmt" + PG_QUERY_COPYSTSMT_NODE = "pg_query.CopyStmt" ) // function type for processing nodes during traversal From 0351d82d97402e8c0f00af38469269b0c8bfe2ce Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Thu, 26 Dec 2024 18:36:06 +0530 Subject: [PATCH 11/17] Reporting multirange datatypes in analyze schema and assessment report for PG(#2112) --- .../dummy-export-dir/schema/tables/table.sql | 36 ++- .../tests/analyze-schema/expected_issues.json | 60 ++++ migtests/tests/analyze-schema/summary.json | 8 +- .../expectedAssessmentReport.json | 267 +++++++++++++++++- .../pg_assessment_report.sql | 30 ++ yb-voyager/src/query/queryissue/constants.go | 6 +- .../src/query/queryissue/detectors_ddl.go | 8 + yb-voyager/src/query/queryissue/issues_ddl.go | 14 + .../src/query/queryissue/issues_ddl_test.go | 40 +++ yb-voyager/src/srcdb/postgres.go | 4 +- 10 files changed, 457 insertions(+), 16 deletions(-) diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql index 82f7411d3..3d9cddc03 100755 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql @@ -384,4 +384,38 @@ CREATE TABLE public.locations ( created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP ); -CREATE TABLE image (title text, raster lo); \ No newline at end of file +CREATE TABLE image (title text, raster lo); + +-- create table with multirange data types + +-- Create tables with primary keys directly +CREATE TABLE bigint_multirange_table ( + id integer PRIMARY KEY, + value_ranges int8multirange +); + +CREATE TABLE date_multirange_table ( + id integer PRIMARY KEY, + project_dates datemultirange +); + +CREATE TABLE int_multirange_table ( + id integer PRIMARY KEY, + value_ranges int4multirange +); + +CREATE TABLE numeric_multirange_table ( + id integer PRIMARY KEY, + price_ranges nummultirange +); + +CREATE TABLE timestamp_multirange_table ( + id integer PRIMARY KEY, + event_times tsmultirange +); + +CREATE TABLE timestamptz_multirange_table ( + id integer PRIMARY KEY, + global_event_times tstzmultirange +); + diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index 28e11bd8c..a6a63a62b 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -1930,5 +1930,65 @@ "Suggestion": "Large objects are not yet supported in YugabyteDB, no workaround available currently", "GH": "https://github.com/yugabyte/yugabyte-db/issues/25318", "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_datatypes", + "ObjectType": "TABLE", + "ObjectName": "bigint_multirange_table", + "Reason": "Unsupported datatype - int8multirange on column - value_ranges", + "SqlStatement": "CREATE TABLE bigint_multirange_table (\n id integer PRIMARY KEY,\n value_ranges int8multirange\n);", + "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_datatypes", + "ObjectType": "TABLE", + "ObjectName": "date_multirange_table", + "Reason": "Unsupported datatype - datemultirange on column - project_dates", + "SqlStatement": "CREATE TABLE date_multirange_table (\n id integer PRIMARY KEY,\n project_dates datemultirange\n);", + "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_datatypes", + "ObjectType": "TABLE", + "ObjectName": "int_multirange_table", + "Reason": "Unsupported datatype - int4multirange on column - value_ranges", + "SqlStatement": "CREATE TABLE int_multirange_table (\n id integer PRIMARY KEY,\n value_ranges int4multirange\n);", + "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_datatypes", + "ObjectType": "TABLE", + "ObjectName": "numeric_multirange_table", + "Reason": "Unsupported datatype - nummultirange on column - price_ranges", + "SqlStatement": "CREATE TABLE numeric_multirange_table (\n id integer PRIMARY KEY,\n price_ranges nummultirange\n);", + "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_datatypes", + "ObjectType": "TABLE", + "ObjectName": "timestamp_multirange_table", + "Reason": "Unsupported datatype - tsmultirange on column - event_times", + "SqlStatement": "CREATE TABLE timestamp_multirange_table (\n id integer PRIMARY KEY,\n event_times tsmultirange\n);", + "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_datatypes", + "ObjectType": "TABLE", + "ObjectName": "timestamptz_multirange_table", + "Reason": "Unsupported datatype - tstzmultirange on column - global_event_times", + "SqlStatement": "CREATE TABLE timestamptz_multirange_table (\n id integer PRIMARY KEY,\n global_event_times tstzmultirange\n);", + "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", + "GH": "", + "MinimumVersionsFixedIn": null } ] diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index f2d1304f3..bdc832263 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -26,9 +26,9 @@ }, { "ObjectType": "TABLE", - "TotalCount": 51, - "InvalidCount": 43, - "ObjectNames": "image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr" }, + "TotalCount": 57, + "InvalidCount": 49, + "ObjectNames": "image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table" }, { "ObjectType": "INDEX", "TotalCount": 43, @@ -91,4 +91,4 @@ "ObjectNames": "\u003c%" } ] -} \ No newline at end of file +} diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 5a861d641..9ac91b514 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -38,15 +38,15 @@ }, { "ObjectType": "SEQUENCE", - "TotalCount": 29, + "TotalCount": 41, "InvalidCount": 0, - "ObjectNames": "public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.employees2_id_seq, public.employees3_id_seq, public.employees_employee_id_seq, public.ext_test_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" + "ObjectNames": "public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.bigint_multirange_table_id_seq, public.date_multirange_table_id_seq, public.employees2_id_seq, public.employees3_id_seq, public.employees_employee_id_seq, public.ext_test_id_seq, public.int_multirange_table_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.numeric_multirange_table_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.timestamp_multirange_table_id_seq, public.timestamptz_multirange_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.bigint_multirange_table_id_seq, schema2.date_multirange_table_id_seq, schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.int_multirange_table_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.numeric_multirange_table_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.timestamp_multirange_table_id_seq, schema2.timestamptz_multirange_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" }, { "ObjectType": "TABLE", - "TotalCount": 68, - "InvalidCount": 23, - "ObjectNames": "public.employees3, public.ordersentry, public.library_nested, public.orders_lateral, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.sales_region, public.boston, public.c, public.parent_table, public.child_table, public.citext_type, public.combined_tbl, public.documents, public.employees2, public.ext_test, public.foo, public.inet_type, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.orders, public.orders2, public.products, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.sales_region, schema2.boston, schema2.c, schema2.parent_table, schema2.child_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.orders, schema2.orders2, schema2.products, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2, public.employees" + "TotalCount": 80, + "InvalidCount": 35, + "ObjectNames": "public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employees3, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" }, { "ObjectType": "INDEX", @@ -171,9 +171,21 @@ "public.library_nested", "public.orders_lateral", "public.employees", - "public.employees3" + "public.employees3", + "public.bigint_multirange_table", + "public.date_multirange_table", + "public.int_multirange_table", + "public.numeric_multirange_table", + "public.timestamp_multirange_table", + "public.timestamptz_multirange_table", + "schema2.bigint_multirange_table", + "schema2.date_multirange_table", + "schema2.int_multirange_table", + "schema2.numeric_multirange_table", + "schema2.timestamp_multirange_table", + "schema2.timestamptz_multirange_table" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 74 objects (66 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 86 objects (78 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": [ "public.combined_tbl", "public.citext_type", @@ -269,6 +281,78 @@ "TableName": "orders_lateral", "ColumnName": "order_details", "DataType": "xml" + }, + { + "SchemaName": "public", + "TableName": "date_multirange_table", + "ColumnName": "project_dates", + "DataType": "datemultirange" + }, + { + "SchemaName": "public", + "TableName": "numeric_multirange_table", + "ColumnName": "price_ranges", + "DataType": "nummultirange" + }, + { + "SchemaName": "public", + "TableName": "bigint_multirange_table", + "ColumnName": "value_ranges", + "DataType": "int8multirange" + }, + { + "SchemaName": "public", + "TableName": "int_multirange_table", + "ColumnName": "value_ranges", + "DataType": "int4multirange" + }, + { + "SchemaName": "public", + "TableName": "timestamp_multirange_table", + "ColumnName": "event_times", + "DataType": "tsmultirange" + }, + { + "SchemaName": "public", + "TableName": "timestamptz_multirange_table", + "ColumnName": "global_event_times", + "DataType": "tstzmultirange" + }, + { + "SchemaName": "schema2", + "TableName": "date_multirange_table", + "ColumnName": "project_dates", + "DataType": "datemultirange" + }, + { + "SchemaName": "schema2", + "TableName": "bigint_multirange_table", + "ColumnName": "value_ranges", + "DataType": "int8multirange" + }, + { + "SchemaName": "schema2", + "TableName": "int_multirange_table", + "ColumnName": "value_ranges", + "DataType": "int4multirange" + }, + { + "SchemaName": "schema2", + "TableName": "numeric_multirange_table", + "ColumnName": "price_ranges", + "DataType": "nummultirange" + }, + { + "SchemaName": "schema2", + "TableName": "timestamp_multirange_table", + "ColumnName": "event_times", + "DataType": "tsmultirange" + }, + { + "SchemaName": "schema2", + "TableName": "timestamptz_multirange_table", + "ColumnName": "global_event_times", + "DataType": "tstzmultirange" } ], "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", @@ -1998,7 +2082,176 @@ "ObjectType": "", "ParentTableName": null, "SizeInBytes": 8192 + }, + { + "SchemaName": "public", + "ObjectName": "bigint_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "public", + "ObjectName": "date_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "public", + "ObjectName": "int_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "public", + "ObjectName": "numeric_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "public", + "ObjectName": "timestamp_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "public", + "ObjectName": "timestamptz_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "schema2", + "ObjectName": "bigint_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "schema2", + "ObjectName": "date_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "schema2", + "ObjectName": "int_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "schema2", + "ObjectName": "numeric_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "schema2", + "ObjectName": "timestamp_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "schema2", + "ObjectName": "timestamptz_multirange_table", + "RowCount": null, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 } + ], "Notes": [ "There are some Unlogged tables in the schema. They will be created as regular LOGGED tables in YugabyteDB as unlogged tables are not supported." diff --git a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql index c23e2ddc5..e1411914d 100644 --- a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql +++ b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql @@ -24,6 +24,36 @@ CREATE TABLE Mixed_Data_Types_Table2 ( path_data PATH ); +CREATE TABLE int_multirange_table ( + id SERIAL PRIMARY KEY, + value_ranges int4multirange +); + +CREATE TABLE bigint_multirange_table ( + id SERIAL PRIMARY KEY, + value_ranges int8multirange +); + +CREATE TABLE numeric_multirange_table ( + id SERIAL PRIMARY KEY, + price_ranges nummultirange +); + +CREATE TABLE timestamp_multirange_table ( + id SERIAL PRIMARY KEY, + event_times tsmultirange +); + +CREATE TABLE timestamptz_multirange_table ( + id SERIAL PRIMARY KEY, + global_event_times tstzmultirange +); + +CREATE TABLE date_multirange_table ( + id SERIAL PRIMARY KEY, + project_dates datemultirange +); + -- GIST Index on point_data column CREATE INDEX idx_point_data ON Mixed_Data_Types_Table1 USING GIST (point_data); diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index 8d319777e..329c25658 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -70,8 +70,10 @@ const ( XML_FUNCTIONS_NAME = "XML Functions" REGEX_FUNCTIONS = "REGEX_FUNCTIONS" - COPY_FROM_WHERE = "COPY FROM ... WHERE" - COPY_ON_ERROR = "COPY ... ON_ERROR" + + MULTI_RANGE_DATATYPE = "MULTI_RANGE_DATATYPE" + COPY_FROM_WHERE = "COPY FROM ... WHERE" + COPY_ON_ERROR = "COPY ... ON_ERROR" ) // Object types diff --git a/yb-voyager/src/query/queryissue/detectors_ddl.go b/yb-voyager/src/query/queryissue/detectors_ddl.go index 82ea56d04..595680f9a 100644 --- a/yb-voyager/src/query/queryissue/detectors_ddl.go +++ b/yb-voyager/src/query/queryissue/detectors_ddl.go @@ -236,6 +236,14 @@ func reportUnsupportedDatatypes(col queryparser.TableColumn, objType string, obj "", col.ColumnName, )) + case "int8multirange", "int4multirange", "datemultirange", "nummultirange", "tsmultirange", "tstzmultirange": + *issues = append(*issues, NewMultiRangeDatatypeIssue( + objType, + objName, + "", + col.TypeName, + col.ColumnName, + )) default: *issues = append(*issues, NewUnsupportedDatatypesIssue( objType, diff --git a/yb-voyager/src/query/queryissue/issues_ddl.go b/yb-voyager/src/query/queryissue/issues_ddl.go index 914dcd536..ba1d4cdae 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl.go +++ b/yb-voyager/src/query/queryissue/issues_ddl.go @@ -440,6 +440,20 @@ func NewLODatatypeIssue(objectType string, objectName string, SqlStatement strin return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) } +var multiRangeDatatypeIssue = issue.Issue{ + Type: MULTI_RANGE_DATATYPE, + TypeName: "Unsupported datatype", + Suggestion: "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", + GH: "", //TODO + DocsLink: "", //TODO +} + +func NewMultiRangeDatatypeIssue(objectType string, objectName string, sqlStatement string, typeName string, colName string) QueryIssue { + issue := multiRangeDatatypeIssue + issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) + return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} + var securityInvokerViewIssue = issue.Issue{ Type: SECURITY_INVOKER_VIEWS, TypeName: "Security Invoker Views not supported yet", diff --git a/yb-voyager/src/query/queryissue/issues_ddl_test.go b/yb-voyager/src/query/queryissue/issues_ddl_test.go index 68993779a..46e913bd1 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl_test.go +++ b/yb-voyager/src/query/queryissue/issues_ddl_test.go @@ -216,6 +216,44 @@ func testLoDatatypeIssue(t *testing.T) { assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "does not exist", loDatatypeIssue) } +func testMultiRangeDatatypeIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + queries := []string{ + `CREATE TABLE int_multirange_table ( + id SERIAL PRIMARY KEY, + value_ranges int4multirange + );`, + `CREATE TABLE bigint_multirange_table ( + id SERIAL PRIMARY KEY, + value_ranges int8multirange + );`, + `CREATE TABLE numeric_multirange_table ( + id SERIAL PRIMARY KEY, + price_ranges nummultirange + );`, + `CREATE TABLE timestamp_multirange_table ( + id SERIAL PRIMARY KEY, + event_times tsmultirange + );`, + `CREATE TABLE timestamptz_multirange_table ( + id SERIAL PRIMARY KEY, + global_event_times tstzmultirange + );`, + `CREATE TABLE date_multirange_table ( + id SERIAL PRIMARY KEY, + project_dates datemultirange + );`, + } + + for _, query := range queries { + _, err = conn.Exec(ctx, query) + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "does not exist", multiRangeDatatypeIssue) + } +} + func testSecurityInvokerView(t *testing.T) { ctx := context.Background() conn, err := getConn() @@ -291,6 +329,8 @@ func TestDDLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "lo datatype", ybVersion), testLoDatatypeIssue) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "multi range datatype", ybVersion), testMultiRangeDatatypeIssue) + assert.True(t, success) success = t.Run(fmt.Sprintf("%s-%s", "security invoker view", ybVersion), testSecurityInvokerView) assert.True(t, success) } diff --git a/yb-voyager/src/srcdb/postgres.go b/yb-voyager/src/srcdb/postgres.go index ba2e21b72..5d7a83813 100644 --- a/yb-voyager/src/srcdb/postgres.go +++ b/yb-voyager/src/srcdb/postgres.go @@ -50,8 +50,8 @@ const PG_STAT_STATEMENTS = "pg_stat_statements" var pg_catalog_tables_required = []string{"regclass", "pg_class", "pg_inherits", "setval", "pg_index", "pg_relation_size", "pg_namespace", "pg_tables", "pg_sequences", "pg_roles", "pg_database", "pg_extension"} var information_schema_tables_required = []string{"schemata", "tables", "columns", "key_column_usage", "sequences"} -var PostgresUnsupportedDataTypes = []string{"GEOMETRY", "GEOGRAPHY", "BOX2D", "BOX3D", "TOPOGEOMETRY", "RASTER", "PG_LSN", "TXID_SNAPSHOT", "XML", "XID", "LO"} -var PostgresUnsupportedDataTypesForDbzm = []string{"POINT", "LINE", "LSEG", "BOX", "PATH", "POLYGON", "CIRCLE", "GEOMETRY", "GEOGRAPHY", "BOX2D", "BOX3D", "TOPOGEOMETRY", "RASTER", "PG_LSN", "TXID_SNAPSHOT", "XML", "LO"} +var PostgresUnsupportedDataTypes = []string{"GEOMETRY", "GEOGRAPHY", "BOX2D", "BOX3D", "TOPOGEOMETRY", "RASTER", "PG_LSN", "TXID_SNAPSHOT", "XML", "XID", "LO", "INT4MULTIRANGE", "INT8MULTIRANGE", "NUMMULTIRANGE", "TSMULTIRANGE", "TSTZMULTIRANGE", "DATEMULTIRANGE"} +var PostgresUnsupportedDataTypesForDbzm = []string{"POINT", "LINE", "LSEG", "BOX", "PATH", "POLYGON", "CIRCLE", "GEOMETRY", "GEOGRAPHY", "BOX2D", "BOX3D", "TOPOGEOMETRY", "RASTER", "PG_LSN", "TXID_SNAPSHOT", "XML", "LO", "INT4MULTIRANGE", "INT8MULTIRANGE", "NUMMULTIRANGE", "TSMULTIRANGE", "TSTZMULTIRANGE", "DATEMULTIRANGE"} func GetPGLiveMigrationUnsupportedDatatypes() []string { liveMigrationUnsupportedDataTypes, _ := lo.Difference(PostgresUnsupportedDataTypesForDbzm, PostgresUnsupportedDataTypes) From bbfc84cd40ecbe3a570077181b1281014b77eed1 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Fri, 27 Dec 2024 12:54:27 +0530 Subject: [PATCH 12/17] Adding build tags to each test file (#2116) - introduced three build tags - unit, integration, issues_integration - every new test file added should have one of these tags - any untagged test file will be executed with every go test command --- .github/workflows/go.yml | 2 +- .github/workflows/integration-tests.yml | 2 +- .github/workflows/issue-tests.yml | 2 +- yb-voyager/cmd/analyzeSchema_test.go | 2 ++ .../cmd/assessMigrationBulkCommand_test.go | 2 ++ yb-voyager/cmd/common_test.go | 17 +++++++++++++++++ yb-voyager/cmd/exportDataStatus_test.go | 17 +++++++++++++++++ yb-voyager/cmd/exportSchema_test.go | 5 ++++- yb-voyager/cmd/import_data_test.go | 17 +++++++++++++++++ yb-voyager/cmd/live_migration_test.go | 2 ++ .../adaptive_parallelism_test.go | 2 ++ yb-voyager/src/callhome/diagnostics_test.go | 17 +++++++++++++++++ yb-voyager/src/cp/yugabyted/yugabyted_test.go | 17 +++++++++++++++++ yb-voyager/src/datafile/descriptor_test.go | 18 ++++++++++++++++++ yb-voyager/src/dbzm/status_test.go | 17 +++++++++++++++++ yb-voyager/src/issue/issue_test.go | 3 ++- yb-voyager/src/metadb/metadataDB_test.go | 17 +++++++++++++++++ .../src/migassessment/assessmentDB_test.go | 17 +++++++++++++++++ yb-voyager/src/migassessment/sizing_test.go | 2 ++ yb-voyager/src/namereg/namereg_test.go | 17 +++++++++++++++++ .../src/query/queryissue/detectors_test.go | 2 ++ .../src/query/queryissue/issues_ddl_test.go | 15 ++++++--------- .../src/query/queryissue/issues_dml_test.go | 5 ++++- .../queryissue/parser_issue_detector_test.go | 17 ++++++++++------- .../src/query/queryparser/traversal_test.go | 2 ++ .../src/tgtdb/attr_name_registry_test.go | 17 +++++++++++++++++ yb-voyager/src/tgtdb/conn_pool_test.go | 2 ++ yb-voyager/src/tgtdb/main_test.go | 2 ++ yb-voyager/src/tgtdb/postgres_test.go | 2 ++ .../src/tgtdb/suites/yugabytedbSuite_test.go | 2 ++ yb-voyager/src/tgtdb/yugabytedb_test.go | 2 ++ yb-voyager/src/utils/jsonfile/jsonfile_test.go | 17 +++++++++++++++++ yb-voyager/src/utils/sqlname/nametuple_test.go | 2 ++ yb-voyager/src/utils/struct_map_test.go | 17 +++++++++++++++++ yb-voyager/src/ybversion/yb_version_test.go | 2 ++ yb-voyager/test/utils/testutils.go | 6 ++++++ 36 files changed, 285 insertions(+), 22 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index ea81fd4e7..88c20428d 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -26,7 +26,7 @@ jobs: - name: Test run: | cd yb-voyager - go test -v -skip '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... -tags '!integration' + go test -v ./... -tags 'unit' - name: Vet run: | diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index b6b018be2..d94a594cb 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -50,4 +50,4 @@ jobs: - name: Run Integration Tests run: | cd yb-voyager - go test -v -skip '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... -tags 'integration' + go test -v ./... -tags 'integration' diff --git a/.github/workflows/issue-tests.yml b/.github/workflows/issue-tests.yml index 7db2e6260..550eef3c8 100644 --- a/.github/workflows/issue-tests.yml +++ b/.github/workflows/issue-tests.yml @@ -47,4 +47,4 @@ jobs: - name: Test Issues Against YB Version run: | cd yb-voyager - go test -v -run '^(TestDDLIssuesInYBVersion|TestDMLIssuesInYBVersion)$' ./... -tags '!integration' + go test -v ./... -tags 'issues_integration' diff --git a/yb-voyager/cmd/analyzeSchema_test.go b/yb-voyager/cmd/analyzeSchema_test.go index 1b97697d3..d768b94b0 100644 --- a/yb-voyager/cmd/analyzeSchema_test.go +++ b/yb-voyager/cmd/analyzeSchema_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/cmd/assessMigrationBulkCommand_test.go b/yb-voyager/cmd/assessMigrationBulkCommand_test.go index cfb8b3436..5b887c901 100644 --- a/yb-voyager/cmd/assessMigrationBulkCommand_test.go +++ b/yb-voyager/cmd/assessMigrationBulkCommand_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/cmd/common_test.go b/yb-voyager/cmd/common_test.go index 6a5c46236..49eee13b0 100644 --- a/yb-voyager/cmd/common_test.go +++ b/yb-voyager/cmd/common_test.go @@ -1,3 +1,20 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package cmd import ( diff --git a/yb-voyager/cmd/exportDataStatus_test.go b/yb-voyager/cmd/exportDataStatus_test.go index 692710aac..20142dba4 100644 --- a/yb-voyager/cmd/exportDataStatus_test.go +++ b/yb-voyager/cmd/exportDataStatus_test.go @@ -1,3 +1,20 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package cmd import ( diff --git a/yb-voyager/cmd/exportSchema_test.go b/yb-voyager/cmd/exportSchema_test.go index 432f76333..a399b0e70 100644 --- a/yb-voyager/cmd/exportSchema_test.go +++ b/yb-voyager/cmd/exportSchema_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. @@ -17,9 +19,10 @@ limitations under the License. package cmd import ( - "github.com/stretchr/testify/assert" "strings" "testing" + + "github.com/stretchr/testify/assert" ) func TestShardingRecommendations(t *testing.T) { diff --git a/yb-voyager/cmd/import_data_test.go b/yb-voyager/cmd/import_data_test.go index e8c3a2300..59fccddf3 100644 --- a/yb-voyager/cmd/import_data_test.go +++ b/yb-voyager/cmd/import_data_test.go @@ -1,3 +1,20 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package cmd import ( diff --git a/yb-voyager/cmd/live_migration_test.go b/yb-voyager/cmd/live_migration_test.go index 3ace7907a..81f9da546 100644 --- a/yb-voyager/cmd/live_migration_test.go +++ b/yb-voyager/cmd/live_migration_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/src/adaptiveparallelism/adaptive_parallelism_test.go b/yb-voyager/src/adaptiveparallelism/adaptive_parallelism_test.go index 7b4438d9e..139b443f4 100644 --- a/yb-voyager/src/adaptiveparallelism/adaptive_parallelism_test.go +++ b/yb-voyager/src/adaptiveparallelism/adaptive_parallelism_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/src/callhome/diagnostics_test.go b/yb-voyager/src/callhome/diagnostics_test.go index 9d63b562b..84781c01f 100644 --- a/yb-voyager/src/callhome/diagnostics_test.go +++ b/yb-voyager/src/callhome/diagnostics_test.go @@ -1,3 +1,20 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package callhome import ( diff --git a/yb-voyager/src/cp/yugabyted/yugabyted_test.go b/yb-voyager/src/cp/yugabyted/yugabyted_test.go index 90bf0c5e2..8e94839d0 100644 --- a/yb-voyager/src/cp/yugabyted/yugabyted_test.go +++ b/yb-voyager/src/cp/yugabyted/yugabyted_test.go @@ -1,3 +1,20 @@ +//go:build integration + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package yugabyted import ( diff --git a/yb-voyager/src/datafile/descriptor_test.go b/yb-voyager/src/datafile/descriptor_test.go index 6092aba4c..7c8b5eace 100644 --- a/yb-voyager/src/datafile/descriptor_test.go +++ b/yb-voyager/src/datafile/descriptor_test.go @@ -1,3 +1,21 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package datafile import ( diff --git a/yb-voyager/src/dbzm/status_test.go b/yb-voyager/src/dbzm/status_test.go index 753188dfd..cfd1f4dea 100644 --- a/yb-voyager/src/dbzm/status_test.go +++ b/yb-voyager/src/dbzm/status_test.go @@ -1,3 +1,20 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package dbzm import ( diff --git a/yb-voyager/src/issue/issue_test.go b/yb-voyager/src/issue/issue_test.go index 19ae50de6..4dd58a564 100644 --- a/yb-voyager/src/issue/issue_test.go +++ b/yb-voyager/src/issue/issue_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. @@ -13,7 +15,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - package issue import ( diff --git a/yb-voyager/src/metadb/metadataDB_test.go b/yb-voyager/src/metadb/metadataDB_test.go index b6c012692..bee70480e 100644 --- a/yb-voyager/src/metadb/metadataDB_test.go +++ b/yb-voyager/src/metadb/metadataDB_test.go @@ -1,3 +1,20 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package metadb import ( diff --git a/yb-voyager/src/migassessment/assessmentDB_test.go b/yb-voyager/src/migassessment/assessmentDB_test.go index 854b52c8f..204acbb12 100644 --- a/yb-voyager/src/migassessment/assessmentDB_test.go +++ b/yb-voyager/src/migassessment/assessmentDB_test.go @@ -1,3 +1,20 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package migassessment import ( diff --git a/yb-voyager/src/migassessment/sizing_test.go b/yb-voyager/src/migassessment/sizing_test.go index dff6c5252..f65bfeb5f 100644 --- a/yb-voyager/src/migassessment/sizing_test.go +++ b/yb-voyager/src/migassessment/sizing_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/src/namereg/namereg_test.go b/yb-voyager/src/namereg/namereg_test.go index c785dbff0..f4c1430da 100644 --- a/yb-voyager/src/namereg/namereg_test.go +++ b/yb-voyager/src/namereg/namereg_test.go @@ -1,3 +1,20 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package namereg import ( diff --git a/yb-voyager/src/query/queryissue/detectors_test.go b/yb-voyager/src/query/queryissue/detectors_test.go index 266f053ba..16c136d3b 100644 --- a/yb-voyager/src/query/queryissue/detectors_test.go +++ b/yb-voyager/src/query/queryissue/detectors_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/src/query/queryissue/issues_ddl_test.go b/yb-voyager/src/query/queryissue/issues_ddl_test.go index 46e913bd1..d53896781 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl_test.go +++ b/yb-voyager/src/query/queryissue/issues_ddl_test.go @@ -1,5 +1,8 @@ +//go:build issues_integration + /* Copyright (c) YugabyteDB, Inc. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -24,9 +27,9 @@ import ( "github.com/jackc/pgx/v5" "github.com/stretchr/testify/assert" "github.com/testcontainers/testcontainers-go/modules/yugabytedb" - "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" ) var ( @@ -56,15 +59,9 @@ func getConn() (*pgx.Conn, error) { return conn, nil } -func fatalIfError(t *testing.T, err error) { - if err != nil { - t.Fatalf("error: %v", err) - } -} - func assertErrorCorrectlyThrownForIssueForYBVersion(t *testing.T, execErr error, expectedError string, issue issue.Issue) { isFixed, err := issue.IsFixedIn(testYbVersion) - fatalIfError(t, err) + testutils.FatalIfError(t, err) if isFixed { assert.NoError(t, execErr) @@ -285,7 +282,7 @@ func TestDDLIssuesInYBVersion(t *testing.T) { ybVersionWithoutBuild := strings.Split(ybVersion, "-")[0] testYbVersion, err = ybversion.NewYBVersion(ybVersionWithoutBuild) - fatalIfError(t, err) + testutils.FatalIfError(t, err) testYugabytedbConnStr = os.Getenv("YB_CONN_STR") if testYugabytedbConnStr == "" { diff --git a/yb-voyager/src/query/queryissue/issues_dml_test.go b/yb-voyager/src/query/queryissue/issues_dml_test.go index d24d4f194..8bbda2d34 100644 --- a/yb-voyager/src/query/queryissue/issues_dml_test.go +++ b/yb-voyager/src/query/queryissue/issues_dml_test.go @@ -1,3 +1,5 @@ +//go:build issues_integration + /* Copyright (c) YugabyteDB, Inc. Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,6 +25,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/testcontainers/testcontainers-go/modules/yugabytedb" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" ) @@ -146,7 +149,7 @@ func TestDMLIssuesInYBVersion(t *testing.T) { ybVersionWithoutBuild := strings.Split(ybVersion, "-")[0] testYbVersion, err = ybversion.NewYBVersion(ybVersionWithoutBuild) - fatalIfError(t, err) + testutils.FatalIfError(t, err) testYugabytedbConnStr = os.Getenv("YB_CONN_STR") if testYugabytedbConnStr == "" { diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index 0d6e9d23f..0481f0d35 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. @@ -23,6 +25,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/samber/lo" "github.com/stretchr/testify/assert" + testutils "github.com/yugabyte/yb-voyager/yb-voyager/test/utils" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" ) @@ -308,12 +311,12 @@ func TestUnloggedTableIssueReportedInOlderVersion(t *testing.T) { // Not reported by default issues, err := parserIssueDetector.GetDDLIssues(stmt, ybversion.LatestStable) - fatalIfError(t, err) + testutils.FatalIfError(t, err) assert.Equal(t, 0, len(issues)) // older version should report the issue issues, err = parserIssueDetector.GetDDLIssues(stmt, ybversion.V2024_1_0_0) - fatalIfError(t, err) + testutils.FatalIfError(t, err) assert.Equal(t, 1, len(issues)) assert.True(t, cmp.Equal(issues[0], NewUnloggedTableIssue("TABLE", "tbl_unlog", stmt))) } @@ -488,7 +491,7 @@ func TestSingleXMLIssueIsDetected(t *testing.T) { parserIssueDetector := NewParserIssueDetector() issues, err := parserIssueDetector.getDMLIssues(stmt) - fatalIfError(t, err) + testutils.FatalIfError(t, err) assert.Equal(t, 1, len(issues)) } @@ -543,7 +546,7 @@ FROM books;`, `SELECT id, JSON_VALUE(details, '$.title') AS title FROM books WHERE JSON_EXISTS(details, '$.price ? (@ > $price)' PASSING 30 AS price);`, -`CREATE MATERIALIZED VIEW public.test_jsonb_view AS + `CREATE MATERIALIZED VIEW public.test_jsonb_view AS SELECT id, data->>'name' AS name, @@ -556,7 +559,7 @@ JSON_TABLE(data, '$.skills[*]' skill TEXT PATH '$' ) ) AS jt;`, - `SELECT JSON_ARRAY($1, 12, TRUE, $2) AS json_array;`, + `SELECT JSON_ARRAY($1, 12, TRUE, $2) AS json_array;`, } sqlsWithExpectedIssues := map[string][]QueryIssue{ sqls[0]: []QueryIssue{ @@ -636,14 +639,14 @@ func TestRegexFunctionsIssue(t *testing.T) { for _, stmt := range dmlStmts { issues, err := parserIssueDetector.getDMLIssues(stmt) - fatalIfError(t, err) + testutils.FatalIfError(t, err) assert.Equal(t, 1, len(issues)) assert.Equal(t, NewRegexFunctionsIssue(DML_QUERY_OBJECT_TYPE, "", stmt), issues[0]) } for _, stmt := range ddlStmts { issues, err := parserIssueDetector.getDDLIssues(stmt) - fatalIfError(t, err) + testutils.FatalIfError(t, err) assert.Equal(t, 1, len(issues)) assert.Equal(t, NewRegexFunctionsIssue(TABLE_OBJECT_TYPE, "x", stmt), issues[0]) } diff --git a/yb-voyager/src/query/queryparser/traversal_test.go b/yb-voyager/src/query/queryparser/traversal_test.go index 9e0f73d61..6e6557971 100644 --- a/yb-voyager/src/query/queryparser/traversal_test.go +++ b/yb-voyager/src/query/queryparser/traversal_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/src/tgtdb/attr_name_registry_test.go b/yb-voyager/src/tgtdb/attr_name_registry_test.go index 8b656336f..3655e477d 100644 --- a/yb-voyager/src/tgtdb/attr_name_registry_test.go +++ b/yb-voyager/src/tgtdb/attr_name_registry_test.go @@ -1,3 +1,20 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package tgtdb import ( diff --git a/yb-voyager/src/tgtdb/conn_pool_test.go b/yb-voyager/src/tgtdb/conn_pool_test.go index 7d4ce4380..55053950d 100644 --- a/yb-voyager/src/tgtdb/conn_pool_test.go +++ b/yb-voyager/src/tgtdb/conn_pool_test.go @@ -1,3 +1,5 @@ +//go:build integration + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/src/tgtdb/main_test.go b/yb-voyager/src/tgtdb/main_test.go index 46cc2150e..ea9539148 100644 --- a/yb-voyager/src/tgtdb/main_test.go +++ b/yb-voyager/src/tgtdb/main_test.go @@ -1,3 +1,5 @@ +//go:build integration + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/src/tgtdb/postgres_test.go b/yb-voyager/src/tgtdb/postgres_test.go index e0e9bd60c..9bb3d563e 100644 --- a/yb-voyager/src/tgtdb/postgres_test.go +++ b/yb-voyager/src/tgtdb/postgres_test.go @@ -1,3 +1,5 @@ +//go:build integration + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/src/tgtdb/suites/yugabytedbSuite_test.go b/yb-voyager/src/tgtdb/suites/yugabytedbSuite_test.go index 834263a61..8f51d621b 100644 --- a/yb-voyager/src/tgtdb/suites/yugabytedbSuite_test.go +++ b/yb-voyager/src/tgtdb/suites/yugabytedbSuite_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/src/tgtdb/yugabytedb_test.go b/yb-voyager/src/tgtdb/yugabytedb_test.go index 755ed46d6..e310877a9 100644 --- a/yb-voyager/src/tgtdb/yugabytedb_test.go +++ b/yb-voyager/src/tgtdb/yugabytedb_test.go @@ -1,3 +1,5 @@ +//go:build integration + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/src/utils/jsonfile/jsonfile_test.go b/yb-voyager/src/utils/jsonfile/jsonfile_test.go index 881873433..a0f20b57e 100644 --- a/yb-voyager/src/utils/jsonfile/jsonfile_test.go +++ b/yb-voyager/src/utils/jsonfile/jsonfile_test.go @@ -1,3 +1,20 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package jsonfile import ( diff --git a/yb-voyager/src/utils/sqlname/nametuple_test.go b/yb-voyager/src/utils/sqlname/nametuple_test.go index 5e7dc33eb..b89ade4c8 100644 --- a/yb-voyager/src/utils/sqlname/nametuple_test.go +++ b/yb-voyager/src/utils/sqlname/nametuple_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/src/utils/struct_map_test.go b/yb-voyager/src/utils/struct_map_test.go index 788251706..c41f90a7d 100644 --- a/yb-voyager/src/utils/struct_map_test.go +++ b/yb-voyager/src/utils/struct_map_test.go @@ -1,3 +1,20 @@ +//go:build unit + +/* +Copyright (c) YugabyteDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package utils import ( diff --git a/yb-voyager/src/ybversion/yb_version_test.go b/yb-voyager/src/ybversion/yb_version_test.go index 0e5352675..e5e67b04e 100644 --- a/yb-voyager/src/ybversion/yb_version_test.go +++ b/yb-voyager/src/ybversion/yb_version_test.go @@ -1,3 +1,5 @@ +//go:build unit + /* Copyright (c) YugabyteDB, Inc. diff --git a/yb-voyager/test/utils/testutils.go b/yb-voyager/test/utils/testutils.go index 74c30bc44..942e00fd6 100644 --- a/yb-voyager/test/utils/testutils.go +++ b/yb-voyager/test/utils/testutils.go @@ -400,3 +400,9 @@ func WaitForDBToBeReady(db *sql.DB) error { } return fmt.Errorf("database did not become ready in time") } + +func FatalIfError(t *testing.T, err error) { + if err != nil { + t.Fatalf("error: %v", err) + } +} From d148a0475e1e5e63cd9ec435090bdc27f2a3d4a9 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Fri, 27 Dec 2024 14:13:58 +0530 Subject: [PATCH 13/17] Regression: Unit test failing due to recent PR merging using code changed in bbfc84cd40ecbe3a570077181b1281014b77eed1 (#2124) - fatalIfError was renamed to testutils.FatalIfError()in this commit, but recent commit was still using old name and git merge was unable to detect this conflict. --- yb-voyager/src/query/queryissue/parser_issue_detector_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index 0481f0d35..e4c6f09d1 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -681,7 +681,7 @@ func TestCopyUnsupportedConstructIssuesDetected(t *testing.T) { for stmt, expectedIssues := range expectedIssues { issues, err := parserIssueDetector.getDMLIssues(stmt) - fatalIfError(t, err) + testutils.FatalIfError(t, err) assert.Equal(t, len(expectedIssues), len(issues)) for _, expectedIssue := range expectedIssues { found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { From 47a8e5088c25f0673c85f4813e273eba841acde9 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Fri, 27 Dec 2024 15:30:36 +0530 Subject: [PATCH 14/17] Upgrade to PG17 in Github actions (#2114) Fixed Validation files for the VIEW/MVIEW's select statements Skipping the SET transaction_timeout = 0; in the import schema to be not executed on YB. Fixes #2121 --- .github/workflows/misc-migtests.yml | 10 +- .github/workflows/pg-13-migtests.yml | 219 ++++++++++++++++++ .../{pg-migtests.yml => pg-17-migtests.yml} | 20 +- .github/workflows/pg-9-migtests.yml | 6 +- migtests/scripts/postgresql/env.sh | 2 +- migtests/scripts/postgresql/ff_env.sh | 2 +- .../expectedAssessmentReport.json | 14 +- .../expected_schema_analysis_report.json | 14 +- .../expectedAssessmentReport.json | 25 +- .../expectedAssessmentReport.json | 12 +- .../expectedAssessmentReport.json | 2 +- .../expected_files/expected_failed.sql | 30 +-- .../expected_schema_analysis_report.json | 2 +- yb-voyager/cmd/constants.go | 3 + yb-voyager/cmd/importSchemaYugabyteDB.go | 29 +-- 15 files changed, 320 insertions(+), 70 deletions(-) create mode 100644 .github/workflows/pg-13-migtests.yml rename .github/workflows/{pg-migtests.yml => pg-17-migtests.yml} (94%) diff --git a/.github/workflows/misc-migtests.yml b/.github/workflows/misc-migtests.yml index cd9fe7ad3..3385743b8 100644 --- a/.github/workflows/misc-migtests.yml +++ b/.github/workflows/misc-migtests.yml @@ -12,9 +12,9 @@ jobs: services: postgres: - image: postgres:15 + image: postgres:17 env: - POSTGRES_PASSWORD: secret + POSTGRES_PASSWORD: postgres # Set health checks to wait until postgres has started options: >- --health-cmd pg_isready @@ -55,17 +55,19 @@ jobs: sudo apt install -y libpq-dev sudo apt install python3-psycopg2 + #TODO Remove the install PG 17 command once we do that in installer script - name: Run installer script to setup voyager run: | cd installer_scripts yes | ./install-yb-voyager --install-from-local-source --only-pg-support - echo "/usr/lib/postgresql/16/bin" >> "$GITHUB_PATH" + sudo apt-get -y install postgresql-17 + echo "/usr/lib/postgresql/17/bin" >> "$GITHUB_PATH" env: ON_INSTALLER_ERROR_OUTPUT_LOG: Y - name: Test PostgreSQL Connection run: | - psql "postgresql://postgres:secret@127.0.0.1:5432/postgres" -c "SELECT version();" + psql "postgresql://postgres:postgres@127.0.0.1:5432/postgres" -c "SELECT version();" - name: Create PostgreSQL user run: | diff --git a/.github/workflows/pg-13-migtests.yml b/.github/workflows/pg-13-migtests.yml new file mode 100644 index 000000000..a64d351fa --- /dev/null +++ b/.github/workflows/pg-13-migtests.yml @@ -0,0 +1,219 @@ +name: "PG 13: Migration Tests" + +on: + push: + branches: ['main', '*.*-dev', '*.*.*-dev'] + pull_request: + branches: ['main'] + +jobs: + run-pg-13-migration-tests: + strategy: + matrix: + version: [2024.2.0.0-b145, 2.20.8.0-b53, 2024.1.3.1-b8, 2.23.1.0-b220] + BETA_FAST_DATA_EXPORT: [0, 1] + test_group: + - offline + - live_basic + - live_advanced + + env: + BETA_FAST_DATA_EXPORT: ${{ matrix.BETA_FAST_DATA_EXPORT }} + runs-on: ubuntu-22.04 + services: + postgres: + image: postgres:13 + env: + POSTGRES_PASSWORD: postgres + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 + # https://github.com/actions/setup-java + with: + distribution: "temurin" + java-version: "17" + check-latest: true + + - name: Cache local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: "Enable postgres with wal_level as logical and install postgis" + run: | + docker exec ${{ job.services.postgres.id }} sh -c "echo 'wal_level=logical' >> /var/lib/postgresql/data/postgresql.conf" + docker exec ${{ job.services.postgres.id }} sh -c "apt-get update && apt-get install -y postgresql-13-postgis postgresql-13-postgis-3" + docker restart ${{ job.services.postgres.id }} + sleep 10 + + - name: Install python3 and psycopg2 + run: | + sudo apt install -y python3 + sudo apt install -y libpq-dev + sudo apt install python3-psycopg2 + + - name: Run installer script to setup voyager + run: | + cd installer_scripts + yes | ./install-yb-voyager --install-from-local-source --only-pg-support + sudo rm /usr/bin/pg_dump + sudo ln -s /usr/lib/postgresql/16/bin/pg_dump /usr/bin/pg_dump + sudo rm /usr/bin/pg_restore + sudo ln -s /usr/lib/postgresql/16/bin/pg_restore /usr/bin/pg_restore + pg_dump --version + env: + ON_INSTALLER_ERROR_OUTPUT_LOG: Y + + - name: Test PostgreSQL Connection + run: | + psql "postgresql://postgres:postgres@127.0.0.1:5432/postgres" -c "SELECT version();" + + - name: Create PostgreSQL user + run: | + ./migtests/scripts/postgresql/create_pg_user + + - name: Start YugabyteDB cluster + run: | + docker run -d --name yugabytedb \ + -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \ + yugabytedb/yugabyte:${{ matrix.version }} \ + bin/yugabyted start --background=false --ui=false + sleep 20 + + - name: Test YugabyteDB connection + run: | + psql "postgresql://yugabyte:@127.0.0.1:5433/yugabyte" -c "SELECT version();" + + - name: Create YugabyteDB user + run: | + ./migtests/scripts/yugabytedb/create_yb_user + + - name: Enable yb-tserver-n1 and yb-master-n1 name resolution + run: | + echo "127.0.0.1 yb-tserver-n1" | sudo tee -a /etc/hosts + echo "127.0.0.1 yb-master-n1" | sudo tee -a /etc/hosts + psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();" + + - name: "TEST: pg-table-list-flags-test (table-list and exclude-table-list)" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/table-list-flags-tests + + - name: "TEST: pg-table-list-file-path-test (table-list-file-path and exclude-table-list-file-path)" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/table-list-flags-tests env-file-path-flags.sh + + - name: "TEST: pg-case-sensitivity-single-table" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test-export-data.sh pg/case-sensitivity-single-table + + - name: "TEST: pg-dvdrental" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/dvdrental + + - name: "TEST: pg-datatypes" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/datatypes + + - name: "TEST: pg-constraints" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/constraints + + - name: "TEST: pg-sequences" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/sequences + + - name: "TEST: pg-indexes" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/indexes + + - name: "TEST: pg-partitions" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/partitions + + - name: "TEST: pg-partitions with (table-list)" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: EXPORT_TABLE_LIST='customers,sales,emp,p2.boston,p2.london,p2.sydney,range_columns_partition_test,sales_region,test_partitions_sequences' migtests/scripts/run-test.sh pg/partitions + + # Broken for v2.15 and v2.16: https://github.com/yugabyte/yugabyte-db/issues/14529 + # Fixed in 2.17.1.0-b368 + - name: "TEST: pg-partitions-with-indexes" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/partitions-with-indexes + + - name: "TEST: pg-views-and-rules" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/views-and-rules + + - name: "TEST: pg-misc-objects-1 (Types, case-sensitive-table-name, Domain)" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/misc-objects-1 + + - name: "TEST: pg-misc-objects-2 (Aggregates, Procedures, triggers, functions, extensions, inline comments)" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/misc-objects-2 + + - name: "TEST: pg-dependent-ddls" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/dependent-ddls + + - name: "TEST: pg-multiple-schemas" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/multiple-schemas + + - name: "TEST: pg-codependent-schemas" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/codependent-schemas + + - name: "TEST: pg-sample-schema-emp" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/sample-employee + + - name: "TEST: pg-hasura-ecommerce" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/hasura-ecommerce + + - name: "TEST: pg-case-sensitivity-reserved-words-offline" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/case-sensitivity-reserved-words + + - name: "TEST: pg-basic-non-public-live-migration-test" + if: ${{ !cancelled() && matrix.test_group == 'live_basic' }} + run: migtests/scripts/live-migration-run-test.sh pg/basic-non-public-live-test + + - name: "TEST: pg-basic-public-fall-forward-test" + if: ${{ !cancelled() && matrix.test_group == 'live_basic' }} + run: migtests/scripts/live-migration-fallf-run-test.sh pg/basic-public-live-test + + # - name: "TEST: pg-basic-non-public-fall-back-test" + # run: migtests/scripts/live-migration-fallb-run-test.sh pg/basic-non-public-live-test + + - name: "TEST: pg-datatypes-fall-back-test" + if: ${{ !cancelled() && matrix.test_group == 'live_basic' }} + run: migtests/scripts/live-migration-fallb-run-test.sh pg/datatypes + + # case sensitive table names are not yet supported in live migration, to restricting test only to a few tables. + - name: "TEST: pg-live-migration-multiple-schemas" + if: ${{ !cancelled() && matrix.test_group == 'live_advanced' }} + run: EXPORT_TABLE_LIST="ext_test,tt,audit,recipients,session_log,schema2.ext_test,schema2.tt,schema2.audit,schema2.recipients,schema2.session_log" migtests/scripts/live-migration-run-test.sh pg/multiple-schemas + + - name: "TEST: pg-unique-key-conflicts-test" + if: ${{ !cancelled() && matrix.test_group == 'live_advanced' }} + run: migtests/scripts/live-migration-fallf-run-test.sh pg/unique-key-conflicts-test + + - name: "TEST: pg-live-migration-partitions-fall-forward" + if: ${{ !cancelled() && matrix.test_group == 'live_advanced' }} + run: migtests/scripts/live-migration-fallf-run-test.sh pg/partitions + diff --git a/.github/workflows/pg-migtests.yml b/.github/workflows/pg-17-migtests.yml similarity index 94% rename from .github/workflows/pg-migtests.yml rename to .github/workflows/pg-17-migtests.yml index 0681dd543..498a4e0cf 100644 --- a/.github/workflows/pg-migtests.yml +++ b/.github/workflows/pg-17-migtests.yml @@ -1,4 +1,4 @@ -name: "PG 13: Migration Tests" +name: "PG 17: Migration Tests" on: push: @@ -7,7 +7,7 @@ on: branches: ['main'] jobs: - run-pg-migration-tests: + run-pg-17-migration-tests: strategy: matrix: version: [2024.2.0.0-b145, 2.20.8.0-b53, 2024.1.3.1-b8, 2.23.1.0-b220] @@ -22,9 +22,9 @@ jobs: runs-on: ubuntu-22.04 services: postgres: - image: postgres:13 + image: postgres:17 env: - POSTGRES_PASSWORD: secret + POSTGRES_PASSWORD: postgres # Set health checks to wait until postgres has started options: >- --health-cmd pg_isready @@ -55,7 +55,7 @@ jobs: - name: "Enable postgres with wal_level as logical and install postgis" run: | docker exec ${{ job.services.postgres.id }} sh -c "echo 'wal_level=logical' >> /var/lib/postgresql/data/postgresql.conf" - docker exec ${{ job.services.postgres.id }} sh -c "apt-get update && apt-get install -y postgresql-13-postgis postgresql-13-postgis-3" + docker exec ${{ job.services.postgres.id }} sh -c "apt-get update && apt-get install -y postgresql-17-postgis postgresql-17-postgis-3" docker restart ${{ job.services.postgres.id }} sleep 10 @@ -65,21 +65,23 @@ jobs: sudo apt install -y libpq-dev sudo apt install python3-psycopg2 + #TODO Remove the install PG 17 command once we do that in installer script - name: Run installer script to setup voyager run: | cd installer_scripts yes | ./install-yb-voyager --install-from-local-source --only-pg-support + sudo apt-get -y install postgresql-17 sudo rm /usr/bin/pg_dump - sudo ln -s /usr/lib/postgresql/16/bin/pg_dump /usr/bin/pg_dump + sudo ln -s /usr/lib/postgresql/17/bin/pg_dump /usr/bin/pg_dump sudo rm /usr/bin/pg_restore - sudo ln -s /usr/lib/postgresql/16/bin/pg_restore /usr/bin/pg_restore + sudo ln -s /usr/lib/postgresql/17/bin/pg_restore /usr/bin/pg_restore pg_dump --version env: ON_INSTALLER_ERROR_OUTPUT_LOG: Y - name: Test PostgreSQL Connection run: | - psql "postgresql://postgres:secret@127.0.0.1:5432/postgres" -c "SELECT version();" + psql "postgresql://postgres:postgres@127.0.0.1:5432/postgres" -c "SELECT version();" - name: Create PostgreSQL user run: | @@ -125,7 +127,7 @@ jobs: - name: "TEST: PG sample schemas (pgtbrus)" if: ${{ !cancelled() && matrix.test_group == 'offline' }} - run: migtests/scripts/run-schema-migration.sh pg/pgtbrus\ + run: migtests/scripts/run-schema-migration.sh pg/pgtbrus - name: "TEST: PG sample schemas (stackexchange)" if: ${{ !cancelled() && matrix.test_group == 'offline' }} diff --git a/.github/workflows/pg-9-migtests.yml b/.github/workflows/pg-9-migtests.yml index f7b36652f..ee406af1b 100644 --- a/.github/workflows/pg-9-migtests.yml +++ b/.github/workflows/pg-9-migtests.yml @@ -7,7 +7,7 @@ on: branches: ['main'] jobs: - run-pg-migration-tests: + run-pg-9-migration-tests: strategy: matrix: version: [2024.2.0.0-b145] @@ -19,7 +19,7 @@ jobs: postgres: image: postgres:9 env: - POSTGRES_PASSWORD: secret + POSTGRES_PASSWORD: postgres # Set health checks to wait until postgres has started options: >- --health-cmd pg_isready @@ -72,7 +72,7 @@ jobs: - name: Test PostgreSQL Connection run: | - psql "postgresql://postgres:secret@127.0.0.1:5432/postgres" -c "SELECT version();" + psql "postgresql://postgres:postgres@127.0.0.1:5432/postgres" -c "SELECT version();" - name: Create PostgreSQL user run: | diff --git a/migtests/scripts/postgresql/env.sh b/migtests/scripts/postgresql/env.sh index ec1f12358..fcc466b82 100644 --- a/migtests/scripts/postgresql/env.sh +++ b/migtests/scripts/postgresql/env.sh @@ -3,4 +3,4 @@ export SOURCE_DB_PORT=${SOURCE_DB_PORT:-5432} export SOURCE_DB_USER=${SOURCE_DB_USER:-"ybvoyager"} export SOURCE_DB_PASSWORD=${SOURCE_DB_PASSWORD:-'Test@123#$%^&*()!'} export SOURCE_DB_ADMIN_USER=${SOURCE_DB_ADMIN_USER:-"postgres"} -export SOURCE_DB_ADMIN_PASSWORD=${SOURCE_DB_ADMIN_PASSWORD:-"secret"} +export SOURCE_DB_ADMIN_PASSWORD=${SOURCE_DB_ADMIN_PASSWORD:-"postgres"} diff --git a/migtests/scripts/postgresql/ff_env.sh b/migtests/scripts/postgresql/ff_env.sh index d8b513f84..0f980435c 100644 --- a/migtests/scripts/postgresql/ff_env.sh +++ b/migtests/scripts/postgresql/ff_env.sh @@ -2,4 +2,4 @@ export SOURCE_REPLICA_DB_NAME=${SOURCE_REPLICA_DB_NAME:-"ff_db"} export SOURCE_REPLICA_DB_HOST=${SOURCE_REPLICA_DB_HOST:-"127.0.0.1"} export SOURCE_REPLICA_DB_PORT=${SOURCE_REPLICA_DB_PORT:-"5432"} export SOURCE_REPLICA_DB_USER=${SOURCE_REPLICA_DB_USER:-"postgres"} -export SOURCE_REPLICA_DB_PASSWORD=${SOURCE_REPLICA_DB_PASSWORD:-"secret"} \ No newline at end of file +export SOURCE_REPLICA_DB_PASSWORD=${SOURCE_REPLICA_DB_PASSWORD:-"postgres"} \ No newline at end of file diff --git a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json index 90979b4b3..54dc75799 100755 --- a/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json @@ -593,15 +593,15 @@ "Objects": [ { "ObjectName": "humanresources.vjobcandidate", - "SqlStatement": "CREATE VIEW humanresources.vjobcandidate AS\n SELECT jobcandidate.jobcandidateid,\n jobcandidate.businessentityid,\n ((xpath('/n:Resume/n:Name/n:Name.Prefix/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Prefix\",\n ((xpath('/n:Resume/n:Name/n:Name.First/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.First\",\n ((xpath('/n:Resume/n:Name/n:Name.Middle/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Middle\",\n ((xpath('/n:Resume/n:Name/n:Name.Last/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Last\",\n ((xpath('/n:Resume/n:Name/n:Name.Suffix/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Suffix\",\n ((xpath('/n:Resume/n:Skills/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"Skills\",\n ((xpath('n:Address/n:Addr.Type/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Addr.Type\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.CountryRegion/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.CountryRegion\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.State/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.State\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.City/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.City\",\n ((xpath('n:Address/n:Addr.PostalCode/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(20) AS \"Addr.PostalCode\",\n ((xpath('/n:Resume/n:EMail/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"EMail\",\n ((xpath('/n:Resume/n:WebSite/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"WebSite\",\n jobcandidate.modifieddate\n FROM humanresources.jobcandidate;" + "SqlStatement": "CREATE VIEW humanresources.vjobcandidate AS\n SELECT jobcandidateid,\n businessentityid,\n ((xpath('/n:Resume/n:Name/n:Name.Prefix/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Prefix\",\n ((xpath('/n:Resume/n:Name/n:Name.First/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.First\",\n ((xpath('/n:Resume/n:Name/n:Name.Middle/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Middle\",\n ((xpath('/n:Resume/n:Name/n:Name.Last/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Last\",\n ((xpath('/n:Resume/n:Name/n:Name.Suffix/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Suffix\",\n ((xpath('/n:Resume/n:Skills/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"Skills\",\n ((xpath('n:Address/n:Addr.Type/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Addr.Type\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.CountryRegion/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.CountryRegion\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.State/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.State\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.City/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.City\",\n ((xpath('n:Address/n:Addr.PostalCode/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(20) AS \"Addr.PostalCode\",\n ((xpath('/n:Resume/n:EMail/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"EMail\",\n ((xpath('/n:Resume/n:WebSite/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"WebSite\",\n modifieddate\n FROM humanresources.jobcandidate;" }, { "ObjectName": "humanresources.vjobcandidateeducation", - "SqlStatement": "CREATE VIEW humanresources.vjobcandidateeducation AS\n SELECT jc.jobcandidateid,\n ((xpath('/root/ns:Education/ns:Edu.Level/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Level\",\n (((xpath('/root/ns:Education/ns:Edu.StartDate/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(20))::date AS \"Edu.StartDate\",\n (((xpath('/root/ns:Education/ns:Edu.EndDate/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(20))::date AS \"Edu.EndDate\",\n ((xpath('/root/ns:Education/ns:Edu.Degree/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Degree\",\n ((xpath('/root/ns:Education/ns:Edu.Major/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Major\",\n ((xpath('/root/ns:Education/ns:Edu.Minor/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Minor\",\n ((xpath('/root/ns:Education/ns:Edu.GPA/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(5) AS \"Edu.GPA\",\n ((xpath('/root/ns:Education/ns:Edu.GPAScale/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(5) AS \"Edu.GPAScale\",\n ((xpath('/root/ns:Education/ns:Edu.School/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.School\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.CountryRegion\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.State/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.State\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.City/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.City\"\n FROM ( SELECT unnesting.jobcandidateid,\n ((('\u003croot xmlns:ns=\"http://adventureworks.com\"\u003e'::text || ((unnesting.education)::character varying)::text) || '\u003c/root\u003e'::text))::xml AS doc\n FROM ( SELECT jobcandidate.jobcandidateid,\n unnest(xpath('/ns:Resume/ns:Education'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])) AS education\n FROM humanresources.jobcandidate) unnesting) jc;" + "SqlStatement": "CREATE VIEW humanresources.vjobcandidateeducation AS\n SELECT jobcandidateid,\n ((xpath('/root/ns:Education/ns:Edu.Level/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Level\",\n (((xpath('/root/ns:Education/ns:Edu.StartDate/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(20))::date AS \"Edu.StartDate\",\n (((xpath('/root/ns:Education/ns:Edu.EndDate/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(20))::date AS \"Edu.EndDate\",\n ((xpath('/root/ns:Education/ns:Edu.Degree/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Degree\",\n ((xpath('/root/ns:Education/ns:Edu.Major/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Major\",\n ((xpath('/root/ns:Education/ns:Edu.Minor/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Minor\",\n ((xpath('/root/ns:Education/ns:Edu.GPA/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(5) AS \"Edu.GPA\",\n ((xpath('/root/ns:Education/ns:Edu.GPAScale/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(5) AS \"Edu.GPAScale\",\n ((xpath('/root/ns:Education/ns:Edu.School/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.School\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.CountryRegion\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.State/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.State\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.City/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.City\"\n FROM ( SELECT unnesting.jobcandidateid,\n ((('\u003croot xmlns:ns=\"http://adventureworks.com\"\u003e'::text || ((unnesting.education)::character varying)::text) || '\u003c/root\u003e'::text))::xml AS doc\n FROM ( SELECT jobcandidate.jobcandidateid,\n unnest(xpath('/ns:Resume/ns:Education'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])) AS education\n FROM humanresources.jobcandidate) unnesting) jc;" }, { "ObjectName": "humanresources.vjobcandidateemployment", - "SqlStatement": "CREATE VIEW humanresources.vjobcandidateemployment AS\n SELECT jobcandidate.jobcandidateid,\n ((unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.StartDate/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(20))::date AS \"Emp.StartDate\",\n ((unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.EndDate/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(20))::date AS \"Emp.EndDate\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.OrgName/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(100) AS \"Emp.OrgName\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.JobTitle/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(100) AS \"Emp.JobTitle\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Responsibility/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Responsibility\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.FunctionCategory/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.FunctionCategory\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.IndustryCategory/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.IndustryCategory\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.CountryRegion\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.State/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.State\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.City/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.City\"\n FROM humanresources.jobcandidate;" + "SqlStatement": "CREATE VIEW humanresources.vjobcandidateemployment AS\n SELECT jobcandidateid,\n ((unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.StartDate/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(20))::date AS \"Emp.StartDate\",\n ((unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.EndDate/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(20))::date AS \"Emp.EndDate\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.OrgName/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(100) AS \"Emp.OrgName\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.JobTitle/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(100) AS \"Emp.JobTitle\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Responsibility/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Responsibility\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.FunctionCategory/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.FunctionCategory\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.IndustryCategory/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.IndustryCategory\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.CountryRegion\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.State/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.State\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.City/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.City\"\n FROM humanresources.jobcandidate;" }, { "ObjectName": "person.vadditionalcontactinfo", @@ -609,19 +609,19 @@ }, { "ObjectName": "production.vproductmodelcatalogdescription", - "SqlStatement": "CREATE VIEW production.vproductmodelcatalogdescription AS\n SELECT productmodel.productmodelid,\n productmodel.name,\n ((xpath('/p1:ProductDescription/p1:Summary/html:p/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{html,http://www.w3.org/1999/xhtml}}'::text[]))[1])::character varying AS \"Summary\",\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:Name/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying AS manufacturer,\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:Copyright/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(30) AS copyright,\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:ProductURL/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS producturl,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:WarrantyPeriod/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS warrantyperiod,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:Description/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS warrantydescription,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:NoOfYears/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS noofyears,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:Description/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS maintenancedescription,\n ((xpath('/p1:ProductDescription/p1:Features/wf:wheel/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS wheel,\n ((xpath('/p1:ProductDescription/p1:Features/wf:saddle/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS saddle,\n ((xpath('/p1:ProductDescription/p1:Features/wf:pedal/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS pedal,\n ((xpath('/p1:ProductDescription/p1:Features/wf:BikeFrame/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying AS bikeframe,\n ((xpath('/p1:ProductDescription/p1:Features/wf:crankset/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS crankset,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:Angle/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS pictureangle,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:Size/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS picturesize,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:ProductPhotoID/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS productphotoid,\n ((xpath('/p1:ProductDescription/p1:Specifications/Material/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS material,\n ((xpath('/p1:ProductDescription/p1:Specifications/Color/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS color,\n ((xpath('/p1:ProductDescription/p1:Specifications/ProductLine/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS productline,\n ((xpath('/p1:ProductDescription/p1:Specifications/Style/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS style,\n ((xpath('/p1:ProductDescription/p1:Specifications/RiderExperience/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(1024) AS riderexperience,\n productmodel.rowguid,\n productmodel.modifieddate\n FROM production.productmodel\n WHERE (productmodel.catalogdescription IS NOT NULL);" + "SqlStatement": "CREATE VIEW production.vproductmodelcatalogdescription AS\n SELECT productmodelid,\n name,\n ((xpath('/p1:ProductDescription/p1:Summary/html:p/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{html,http://www.w3.org/1999/xhtml}}'::text[]))[1])::character varying AS \"Summary\",\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:Name/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying AS manufacturer,\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:Copyright/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(30) AS copyright,\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:ProductURL/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS producturl,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:WarrantyPeriod/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS warrantyperiod,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:Description/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS warrantydescription,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:NoOfYears/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS noofyears,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:Description/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS maintenancedescription,\n ((xpath('/p1:ProductDescription/p1:Features/wf:wheel/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS wheel,\n ((xpath('/p1:ProductDescription/p1:Features/wf:saddle/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS saddle,\n ((xpath('/p1:ProductDescription/p1:Features/wf:pedal/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS pedal,\n ((xpath('/p1:ProductDescription/p1:Features/wf:BikeFrame/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying AS bikeframe,\n ((xpath('/p1:ProductDescription/p1:Features/wf:crankset/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS crankset,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:Angle/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS pictureangle,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:Size/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS picturesize,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:ProductPhotoID/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS productphotoid,\n ((xpath('/p1:ProductDescription/p1:Specifications/Material/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS material,\n ((xpath('/p1:ProductDescription/p1:Specifications/Color/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS color,\n ((xpath('/p1:ProductDescription/p1:Specifications/ProductLine/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS productline,\n ((xpath('/p1:ProductDescription/p1:Specifications/Style/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS style,\n ((xpath('/p1:ProductDescription/p1:Specifications/RiderExperience/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(1024) AS riderexperience,\n rowguid,\n modifieddate\n FROM production.productmodel\n WHERE (catalogdescription IS NOT NULL);" }, { "ObjectName": "production.vproductmodelinstructions", - "SqlStatement": "CREATE VIEW production.vproductmodelinstructions AS\n SELECT pm.productmodelid,\n pm.name,\n ((xpath('/ns:root/text()'::text, pm.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[]))[1])::character varying AS instructions,\n (((xpath('@LocationID'::text, pm.mfginstructions))[1])::character varying)::integer AS \"LocationID\",\n (((xpath('@SetupHours'::text, pm.mfginstructions))[1])::character varying)::numeric(9,4) AS \"SetupHours\",\n (((xpath('@MachineHours'::text, pm.mfginstructions))[1])::character varying)::numeric(9,4) AS \"MachineHours\",\n (((xpath('@LaborHours'::text, pm.mfginstructions))[1])::character varying)::numeric(9,4) AS \"LaborHours\",\n (((xpath('@LotSize'::text, pm.mfginstructions))[1])::character varying)::integer AS \"LotSize\",\n ((xpath('/step/text()'::text, pm.step))[1])::character varying(1024) AS \"Step\",\n pm.rowguid,\n pm.modifieddate\n FROM ( SELECT locations.productmodelid,\n locations.name,\n locations.rowguid,\n locations.modifieddate,\n locations.instructions,\n locations.mfginstructions,\n unnest(xpath('step'::text, locations.mfginstructions)) AS step\n FROM ( SELECT productmodel.productmodelid,\n productmodel.name,\n productmodel.rowguid,\n productmodel.modifieddate,\n productmodel.instructions,\n unnest(xpath('/ns:root/ns:Location'::text, productmodel.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[])) AS mfginstructions\n FROM production.productmodel) locations) pm;" + "SqlStatement": "CREATE VIEW production.vproductmodelinstructions AS\n SELECT productmodelid,\n name,\n ((xpath('/ns:root/text()'::text, instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[]))[1])::character varying AS instructions,\n (((xpath('@LocationID'::text, mfginstructions))[1])::character varying)::integer AS \"LocationID\",\n (((xpath('@SetupHours'::text, mfginstructions))[1])::character varying)::numeric(9,4) AS \"SetupHours\",\n (((xpath('@MachineHours'::text, mfginstructions))[1])::character varying)::numeric(9,4) AS \"MachineHours\",\n (((xpath('@LaborHours'::text, mfginstructions))[1])::character varying)::numeric(9,4) AS \"LaborHours\",\n (((xpath('@LotSize'::text, mfginstructions))[1])::character varying)::integer AS \"LotSize\",\n ((xpath('/step/text()'::text, step))[1])::character varying(1024) AS \"Step\",\n rowguid,\n modifieddate\n FROM ( SELECT locations.productmodelid,\n locations.name,\n locations.rowguid,\n locations.modifieddate,\n locations.instructions,\n locations.mfginstructions,\n unnest(xpath('step'::text, locations.mfginstructions)) AS step\n FROM ( SELECT productmodel.productmodelid,\n productmodel.name,\n productmodel.rowguid,\n productmodel.modifieddate,\n productmodel.instructions,\n unnest(xpath('/ns:root/ns:Location'::text, productmodel.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[])) AS mfginstructions\n FROM production.productmodel) locations) pm;" }, { "ObjectName": "sales.vpersondemographics", - "SqlStatement": "CREATE VIEW sales.vpersondemographics AS\n SELECT person.businessentityid,\n (((xpath('n:TotalPurchaseYTD/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::money AS totalpurchaseytd,\n (((xpath('n:DateFirstPurchase/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::date AS datefirstpurchase,\n (((xpath('n:BirthDate/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::date AS birthdate,\n ((xpath('n:MaritalStatus/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(1) AS maritalstatus,\n ((xpath('n:YearlyIncome/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS yearlyincome,\n ((xpath('n:Gender/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(1) AS gender,\n (((xpath('n:TotalChildren/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS totalchildren,\n (((xpath('n:NumberChildrenAtHome/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS numberchildrenathome,\n ((xpath('n:Education/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS education,\n ((xpath('n:Occupation/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS occupation,\n (((xpath('n:HomeOwnerFlag/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::boolean AS homeownerflag,\n (((xpath('n:NumberCarsOwned/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS numbercarsowned\n FROM person.person\n WHERE (person.demographics IS NOT NULL);" + "SqlStatement": "CREATE VIEW sales.vpersondemographics AS\n SELECT businessentityid,\n (((xpath('n:TotalPurchaseYTD/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::money AS totalpurchaseytd,\n (((xpath('n:DateFirstPurchase/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::date AS datefirstpurchase,\n (((xpath('n:BirthDate/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::date AS birthdate,\n ((xpath('n:MaritalStatus/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(1) AS maritalstatus,\n ((xpath('n:YearlyIncome/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS yearlyincome,\n ((xpath('n:Gender/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(1) AS gender,\n (((xpath('n:TotalChildren/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS totalchildren,\n (((xpath('n:NumberChildrenAtHome/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS numberchildrenathome,\n ((xpath('n:Education/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS education,\n ((xpath('n:Occupation/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS occupation,\n (((xpath('n:HomeOwnerFlag/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::boolean AS homeownerflag,\n (((xpath('n:NumberCarsOwned/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS numbercarsowned\n FROM person.person\n WHERE (demographics IS NOT NULL);" }, { "ObjectName": "sales.vstorewithdemographics", - "SqlStatement": "CREATE VIEW sales.vstorewithdemographics AS\n SELECT store.businessentityid,\n store.name,\n ((unnest(xpath('/ns:StoreSurvey/ns:AnnualSales/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::money AS \"AnnualSales\",\n ((unnest(xpath('/ns:StoreSurvey/ns:AnnualRevenue/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::money AS \"AnnualRevenue\",\n (unnest(xpath('/ns:StoreSurvey/ns:BankName/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(50) AS \"BankName\",\n (unnest(xpath('/ns:StoreSurvey/ns:BusinessType/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(5) AS \"BusinessType\",\n ((unnest(xpath('/ns:StoreSurvey/ns:YearOpened/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"YearOpened\",\n (unnest(xpath('/ns:StoreSurvey/ns:Specialty/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(50) AS \"Specialty\",\n ((unnest(xpath('/ns:StoreSurvey/ns:SquareFeet/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"SquareFeet\",\n (unnest(xpath('/ns:StoreSurvey/ns:Brands/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(30) AS \"Brands\",\n (unnest(xpath('/ns:StoreSurvey/ns:Internet/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(30) AS \"Internet\",\n ((unnest(xpath('/ns:StoreSurvey/ns:NumberEmployees/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"NumberEmployees\"\n FROM sales.store;" + "SqlStatement": "CREATE VIEW sales.vstorewithdemographics AS\n SELECT businessentityid,\n name,\n ((unnest(xpath('/ns:StoreSurvey/ns:AnnualSales/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::money AS \"AnnualSales\",\n ((unnest(xpath('/ns:StoreSurvey/ns:AnnualRevenue/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::money AS \"AnnualRevenue\",\n (unnest(xpath('/ns:StoreSurvey/ns:BankName/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(50) AS \"BankName\",\n (unnest(xpath('/ns:StoreSurvey/ns:BusinessType/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(5) AS \"BusinessType\",\n ((unnest(xpath('/ns:StoreSurvey/ns:YearOpened/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"YearOpened\",\n (unnest(xpath('/ns:StoreSurvey/ns:Specialty/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(50) AS \"Specialty\",\n ((unnest(xpath('/ns:StoreSurvey/ns:SquareFeet/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"SquareFeet\",\n (unnest(xpath('/ns:StoreSurvey/ns:Brands/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(30) AS \"Brands\",\n (unnest(xpath('/ns:StoreSurvey/ns:Internet/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(30) AS \"Internet\",\n ((unnest(xpath('/ns:StoreSurvey/ns:NumberEmployees/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"NumberEmployees\"\n FROM sales.store;" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", diff --git a/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json index fa0a6e795..c58df2434 100755 --- a/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/adventureworks/expected_files/expected_schema_analysis_report.json @@ -975,7 +975,7 @@ "ObjectType": "VIEW", "ObjectName": "humanresources.vjobcandidate", "Reason": "XML Functions", - "SqlStatement": "CREATE VIEW humanresources.vjobcandidate AS\n SELECT jobcandidate.jobcandidateid,\n jobcandidate.businessentityid,\n ((xpath('/n:Resume/n:Name/n:Name.Prefix/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Prefix\",\n ((xpath('/n:Resume/n:Name/n:Name.First/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.First\",\n ((xpath('/n:Resume/n:Name/n:Name.Middle/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Middle\",\n ((xpath('/n:Resume/n:Name/n:Name.Last/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Last\",\n ((xpath('/n:Resume/n:Name/n:Name.Suffix/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Suffix\",\n ((xpath('/n:Resume/n:Skills/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"Skills\",\n ((xpath('n:Address/n:Addr.Type/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Addr.Type\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.CountryRegion/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.CountryRegion\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.State/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.State\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.City/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.City\",\n ((xpath('n:Address/n:Addr.PostalCode/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(20) AS \"Addr.PostalCode\",\n ((xpath('/n:Resume/n:EMail/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"EMail\",\n ((xpath('/n:Resume/n:WebSite/text()'::text, jobcandidate.resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"WebSite\",\n jobcandidate.modifieddate\n FROM humanresources.jobcandidate;", + "SqlStatement": "CREATE VIEW humanresources.vjobcandidate AS\n SELECT jobcandidateid,\n businessentityid,\n ((xpath('/n:Resume/n:Name/n:Name.Prefix/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Prefix\",\n ((xpath('/n:Resume/n:Name/n:Name.First/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.First\",\n ((xpath('/n:Resume/n:Name/n:Name.Middle/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Middle\",\n ((xpath('/n:Resume/n:Name/n:Name.Last/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Last\",\n ((xpath('/n:Resume/n:Name/n:Name.Suffix/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Name.Suffix\",\n ((xpath('/n:Resume/n:Skills/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"Skills\",\n ((xpath('n:Address/n:Addr.Type/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(30) AS \"Addr.Type\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.CountryRegion/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.CountryRegion\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.State/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.State\",\n ((xpath('n:Address/n:Addr.Location/n:Location/n:Loc.City/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(100) AS \"Addr.Loc.City\",\n ((xpath('n:Address/n:Addr.PostalCode/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying(20) AS \"Addr.PostalCode\",\n ((xpath('/n:Resume/n:EMail/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"EMail\",\n ((xpath('/n:Resume/n:WebSite/text()'::text, resume, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[]))[1])::character varying AS \"WebSite\",\n modifieddate\n FROM humanresources.jobcandidate;", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", @@ -987,7 +987,7 @@ "ObjectType": "VIEW", "ObjectName": "humanresources.vjobcandidateeducation", "Reason": "XML Functions", - "SqlStatement": "CREATE VIEW humanresources.vjobcandidateeducation AS\n SELECT jc.jobcandidateid,\n ((xpath('/root/ns:Education/ns:Edu.Level/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Level\",\n (((xpath('/root/ns:Education/ns:Edu.StartDate/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(20))::date AS \"Edu.StartDate\",\n (((xpath('/root/ns:Education/ns:Edu.EndDate/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(20))::date AS \"Edu.EndDate\",\n ((xpath('/root/ns:Education/ns:Edu.Degree/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Degree\",\n ((xpath('/root/ns:Education/ns:Edu.Major/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Major\",\n ((xpath('/root/ns:Education/ns:Edu.Minor/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Minor\",\n ((xpath('/root/ns:Education/ns:Edu.GPA/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(5) AS \"Edu.GPA\",\n ((xpath('/root/ns:Education/ns:Edu.GPAScale/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(5) AS \"Edu.GPAScale\",\n ((xpath('/root/ns:Education/ns:Edu.School/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.School\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.CountryRegion\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.State/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.State\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.City/text()'::text, jc.doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.City\"\n FROM ( SELECT unnesting.jobcandidateid,\n ((('\u003croot xmlns:ns=\"http://adventureworks.com\"\u003e'::text || ((unnesting.education)::character varying)::text) || '\u003c/root\u003e'::text))::xml AS doc\n FROM ( SELECT jobcandidate.jobcandidateid,\n unnest(xpath('/ns:Resume/ns:Education'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])) AS education\n FROM humanresources.jobcandidate) unnesting) jc;", + "SqlStatement": "CREATE VIEW humanresources.vjobcandidateeducation AS\n SELECT jobcandidateid,\n ((xpath('/root/ns:Education/ns:Edu.Level/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Level\",\n (((xpath('/root/ns:Education/ns:Edu.StartDate/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(20))::date AS \"Edu.StartDate\",\n (((xpath('/root/ns:Education/ns:Edu.EndDate/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(20))::date AS \"Edu.EndDate\",\n ((xpath('/root/ns:Education/ns:Edu.Degree/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Degree\",\n ((xpath('/root/ns:Education/ns:Edu.Major/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Major\",\n ((xpath('/root/ns:Education/ns:Edu.Minor/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(50) AS \"Edu.Minor\",\n ((xpath('/root/ns:Education/ns:Edu.GPA/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(5) AS \"Edu.GPA\",\n ((xpath('/root/ns:Education/ns:Edu.GPAScale/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(5) AS \"Edu.GPAScale\",\n ((xpath('/root/ns:Education/ns:Edu.School/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.School\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.CountryRegion\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.State/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.State\",\n ((xpath('/root/ns:Education/ns:Edu.Location/ns:Location/ns:Loc.City/text()'::text, doc, '{{ns,http://adventureworks.com}}'::text[]))[1])::character varying(100) AS \"Edu.Loc.City\"\n FROM ( SELECT unnesting.jobcandidateid,\n ((('\u003croot xmlns:ns=\"http://adventureworks.com\"\u003e'::text || ((unnesting.education)::character varying)::text) || '\u003c/root\u003e'::text))::xml AS doc\n FROM ( SELECT jobcandidate.jobcandidateid,\n unnest(xpath('/ns:Resume/ns:Education'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])) AS education\n FROM humanresources.jobcandidate) unnesting) jc;", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", @@ -999,7 +999,7 @@ "ObjectType": "VIEW", "ObjectName": "humanresources.vjobcandidateemployment", "Reason": "XML Functions", - "SqlStatement": "CREATE VIEW humanresources.vjobcandidateemployment AS\n SELECT jobcandidate.jobcandidateid,\n ((unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.StartDate/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(20))::date AS \"Emp.StartDate\",\n ((unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.EndDate/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(20))::date AS \"Emp.EndDate\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.OrgName/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(100) AS \"Emp.OrgName\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.JobTitle/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(100) AS \"Emp.JobTitle\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Responsibility/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Responsibility\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.FunctionCategory/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.FunctionCategory\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.IndustryCategory/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.IndustryCategory\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.CountryRegion\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.State/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.State\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.City/text()'::text, jobcandidate.resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.City\"\n FROM humanresources.jobcandidate;", + "SqlStatement": "CREATE VIEW humanresources.vjobcandidateemployment AS\n SELECT jobcandidateid,\n ((unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.StartDate/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(20))::date AS \"Emp.StartDate\",\n ((unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.EndDate/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(20))::date AS \"Emp.EndDate\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.OrgName/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(100) AS \"Emp.OrgName\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.JobTitle/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying(100) AS \"Emp.JobTitle\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Responsibility/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Responsibility\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.FunctionCategory/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.FunctionCategory\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.IndustryCategory/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.IndustryCategory\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.CountryRegion/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.CountryRegion\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.State/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.State\",\n (unnest(xpath('/ns:Resume/ns:Employment/ns:Emp.Location/ns:Location/ns:Loc.City/text()'::text, resume, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/Resume}}'::text[])))::character varying AS \"Emp.Loc.City\"\n FROM humanresources.jobcandidate;", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", @@ -1023,7 +1023,7 @@ "ObjectType": "VIEW", "ObjectName": "production.vproductmodelcatalogdescription", "Reason": "XML Functions", - "SqlStatement": "CREATE VIEW production.vproductmodelcatalogdescription AS\n SELECT productmodel.productmodelid,\n productmodel.name,\n ((xpath('/p1:ProductDescription/p1:Summary/html:p/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{html,http://www.w3.org/1999/xhtml}}'::text[]))[1])::character varying AS \"Summary\",\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:Name/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying AS manufacturer,\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:Copyright/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(30) AS copyright,\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:ProductURL/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS producturl,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:WarrantyPeriod/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS warrantyperiod,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:Description/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS warrantydescription,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:NoOfYears/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS noofyears,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:Description/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS maintenancedescription,\n ((xpath('/p1:ProductDescription/p1:Features/wf:wheel/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS wheel,\n ((xpath('/p1:ProductDescription/p1:Features/wf:saddle/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS saddle,\n ((xpath('/p1:ProductDescription/p1:Features/wf:pedal/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS pedal,\n ((xpath('/p1:ProductDescription/p1:Features/wf:BikeFrame/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying AS bikeframe,\n ((xpath('/p1:ProductDescription/p1:Features/wf:crankset/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS crankset,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:Angle/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS pictureangle,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:Size/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS picturesize,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:ProductPhotoID/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS productphotoid,\n ((xpath('/p1:ProductDescription/p1:Specifications/Material/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS material,\n ((xpath('/p1:ProductDescription/p1:Specifications/Color/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS color,\n ((xpath('/p1:ProductDescription/p1:Specifications/ProductLine/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS productline,\n ((xpath('/p1:ProductDescription/p1:Specifications/Style/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS style,\n ((xpath('/p1:ProductDescription/p1:Specifications/RiderExperience/text()'::text, productmodel.catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(1024) AS riderexperience,\n productmodel.rowguid,\n productmodel.modifieddate\n FROM production.productmodel\n WHERE (productmodel.catalogdescription IS NOT NULL);", + "SqlStatement": "CREATE VIEW production.vproductmodelcatalogdescription AS\n SELECT productmodelid,\n name,\n ((xpath('/p1:ProductDescription/p1:Summary/html:p/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{html,http://www.w3.org/1999/xhtml}}'::text[]))[1])::character varying AS \"Summary\",\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:Name/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying AS manufacturer,\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:Copyright/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(30) AS copyright,\n ((xpath('/p1:ProductDescription/p1:Manufacturer/p1:ProductURL/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS producturl,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:WarrantyPeriod/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS warrantyperiod,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Warranty/wm:Description/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS warrantydescription,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:NoOfYears/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS noofyears,\n ((xpath('/p1:ProductDescription/p1:Features/wm:Maintenance/wm:Description/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wm,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelWarrAndMain}}'::text[]))[1])::character varying(256) AS maintenancedescription,\n ((xpath('/p1:ProductDescription/p1:Features/wf:wheel/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS wheel,\n ((xpath('/p1:ProductDescription/p1:Features/wf:saddle/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS saddle,\n ((xpath('/p1:ProductDescription/p1:Features/wf:pedal/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS pedal,\n ((xpath('/p1:ProductDescription/p1:Features/wf:BikeFrame/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying AS bikeframe,\n ((xpath('/p1:ProductDescription/p1:Features/wf:crankset/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription},{wf,http://www.adventure-works.com/schemas/OtherFeatures}}'::text[]))[1])::character varying(256) AS crankset,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:Angle/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS pictureangle,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:Size/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS picturesize,\n ((xpath('/p1:ProductDescription/p1:Picture/p1:ProductPhotoID/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS productphotoid,\n ((xpath('/p1:ProductDescription/p1:Specifications/Material/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS material,\n ((xpath('/p1:ProductDescription/p1:Specifications/Color/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS color,\n ((xpath('/p1:ProductDescription/p1:Specifications/ProductLine/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS productline,\n ((xpath('/p1:ProductDescription/p1:Specifications/Style/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(256) AS style,\n ((xpath('/p1:ProductDescription/p1:Specifications/RiderExperience/text()'::text, catalogdescription, '{{p1,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelDescription}}'::text[]))[1])::character varying(1024) AS riderexperience,\n rowguid,\n modifieddate\n FROM production.productmodel\n WHERE (catalogdescription IS NOT NULL);", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", @@ -1035,7 +1035,7 @@ "ObjectType": "VIEW", "ObjectName": "production.vproductmodelinstructions", "Reason": "XML Functions", - "SqlStatement": "CREATE VIEW production.vproductmodelinstructions AS\n SELECT pm.productmodelid,\n pm.name,\n ((xpath('/ns:root/text()'::text, pm.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[]))[1])::character varying AS instructions,\n (((xpath('@LocationID'::text, pm.mfginstructions))[1])::character varying)::integer AS \"LocationID\",\n (((xpath('@SetupHours'::text, pm.mfginstructions))[1])::character varying)::numeric(9,4) AS \"SetupHours\",\n (((xpath('@MachineHours'::text, pm.mfginstructions))[1])::character varying)::numeric(9,4) AS \"MachineHours\",\n (((xpath('@LaborHours'::text, pm.mfginstructions))[1])::character varying)::numeric(9,4) AS \"LaborHours\",\n (((xpath('@LotSize'::text, pm.mfginstructions))[1])::character varying)::integer AS \"LotSize\",\n ((xpath('/step/text()'::text, pm.step))[1])::character varying(1024) AS \"Step\",\n pm.rowguid,\n pm.modifieddate\n FROM ( SELECT locations.productmodelid,\n locations.name,\n locations.rowguid,\n locations.modifieddate,\n locations.instructions,\n locations.mfginstructions,\n unnest(xpath('step'::text, locations.mfginstructions)) AS step\n FROM ( SELECT productmodel.productmodelid,\n productmodel.name,\n productmodel.rowguid,\n productmodel.modifieddate,\n productmodel.instructions,\n unnest(xpath('/ns:root/ns:Location'::text, productmodel.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[])) AS mfginstructions\n FROM production.productmodel) locations) pm;", + "SqlStatement": "CREATE VIEW production.vproductmodelinstructions AS\n SELECT productmodelid,\n name,\n ((xpath('/ns:root/text()'::text, instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[]))[1])::character varying AS instructions,\n (((xpath('@LocationID'::text, mfginstructions))[1])::character varying)::integer AS \"LocationID\",\n (((xpath('@SetupHours'::text, mfginstructions))[1])::character varying)::numeric(9,4) AS \"SetupHours\",\n (((xpath('@MachineHours'::text, mfginstructions))[1])::character varying)::numeric(9,4) AS \"MachineHours\",\n (((xpath('@LaborHours'::text, mfginstructions))[1])::character varying)::numeric(9,4) AS \"LaborHours\",\n (((xpath('@LotSize'::text, mfginstructions))[1])::character varying)::integer AS \"LotSize\",\n ((xpath('/step/text()'::text, step))[1])::character varying(1024) AS \"Step\",\n rowguid,\n modifieddate\n FROM ( SELECT locations.productmodelid,\n locations.name,\n locations.rowguid,\n locations.modifieddate,\n locations.instructions,\n locations.mfginstructions,\n unnest(xpath('step'::text, locations.mfginstructions)) AS step\n FROM ( SELECT productmodel.productmodelid,\n productmodel.name,\n productmodel.rowguid,\n productmodel.modifieddate,\n productmodel.instructions,\n unnest(xpath('/ns:root/ns:Location'::text, productmodel.instructions, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/ProductModelManuInstructions}}'::text[])) AS mfginstructions\n FROM production.productmodel) locations) pm;", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", @@ -1047,7 +1047,7 @@ "ObjectType": "VIEW", "ObjectName": "sales.vpersondemographics", "Reason": "XML Functions", - "SqlStatement": "CREATE VIEW sales.vpersondemographics AS\n SELECT person.businessentityid,\n (((xpath('n:TotalPurchaseYTD/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::money AS totalpurchaseytd,\n (((xpath('n:DateFirstPurchase/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::date AS datefirstpurchase,\n (((xpath('n:BirthDate/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::date AS birthdate,\n ((xpath('n:MaritalStatus/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(1) AS maritalstatus,\n ((xpath('n:YearlyIncome/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS yearlyincome,\n ((xpath('n:Gender/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(1) AS gender,\n (((xpath('n:TotalChildren/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS totalchildren,\n (((xpath('n:NumberChildrenAtHome/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS numberchildrenathome,\n ((xpath('n:Education/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS education,\n ((xpath('n:Occupation/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS occupation,\n (((xpath('n:HomeOwnerFlag/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::boolean AS homeownerflag,\n (((xpath('n:NumberCarsOwned/text()'::text, person.demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS numbercarsowned\n FROM person.person\n WHERE (person.demographics IS NOT NULL);", + "SqlStatement": "CREATE VIEW sales.vpersondemographics AS\n SELECT businessentityid,\n (((xpath('n:TotalPurchaseYTD/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::money AS totalpurchaseytd,\n (((xpath('n:DateFirstPurchase/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::date AS datefirstpurchase,\n (((xpath('n:BirthDate/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::date AS birthdate,\n ((xpath('n:MaritalStatus/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(1) AS maritalstatus,\n ((xpath('n:YearlyIncome/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS yearlyincome,\n ((xpath('n:Gender/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(1) AS gender,\n (((xpath('n:TotalChildren/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS totalchildren,\n (((xpath('n:NumberChildrenAtHome/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS numberchildrenathome,\n ((xpath('n:Education/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS education,\n ((xpath('n:Occupation/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying(30) AS occupation,\n (((xpath('n:HomeOwnerFlag/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::boolean AS homeownerflag,\n (((xpath('n:NumberCarsOwned/text()'::text, demographics, '{{n,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/IndividualSurvey}}'::text[]))[1])::character varying)::integer AS numbercarsowned\n FROM person.person\n WHERE (demographics IS NOT NULL);", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", @@ -1059,7 +1059,7 @@ "ObjectType": "VIEW", "ObjectName": "sales.vstorewithdemographics", "Reason": "XML Functions", - "SqlStatement": "CREATE VIEW sales.vstorewithdemographics AS\n SELECT store.businessentityid,\n store.name,\n ((unnest(xpath('/ns:StoreSurvey/ns:AnnualSales/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::money AS \"AnnualSales\",\n ((unnest(xpath('/ns:StoreSurvey/ns:AnnualRevenue/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::money AS \"AnnualRevenue\",\n (unnest(xpath('/ns:StoreSurvey/ns:BankName/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(50) AS \"BankName\",\n (unnest(xpath('/ns:StoreSurvey/ns:BusinessType/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(5) AS \"BusinessType\",\n ((unnest(xpath('/ns:StoreSurvey/ns:YearOpened/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"YearOpened\",\n (unnest(xpath('/ns:StoreSurvey/ns:Specialty/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(50) AS \"Specialty\",\n ((unnest(xpath('/ns:StoreSurvey/ns:SquareFeet/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"SquareFeet\",\n (unnest(xpath('/ns:StoreSurvey/ns:Brands/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(30) AS \"Brands\",\n (unnest(xpath('/ns:StoreSurvey/ns:Internet/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(30) AS \"Internet\",\n ((unnest(xpath('/ns:StoreSurvey/ns:NumberEmployees/text()'::text, store.demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"NumberEmployees\"\n FROM sales.store;", + "SqlStatement": "CREATE VIEW sales.vstorewithdemographics AS\n SELECT businessentityid,\n name,\n ((unnest(xpath('/ns:StoreSurvey/ns:AnnualSales/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::money AS \"AnnualSales\",\n ((unnest(xpath('/ns:StoreSurvey/ns:AnnualRevenue/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::money AS \"AnnualRevenue\",\n (unnest(xpath('/ns:StoreSurvey/ns:BankName/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(50) AS \"BankName\",\n (unnest(xpath('/ns:StoreSurvey/ns:BusinessType/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(5) AS \"BusinessType\",\n ((unnest(xpath('/ns:StoreSurvey/ns:YearOpened/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"YearOpened\",\n (unnest(xpath('/ns:StoreSurvey/ns:Specialty/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(50) AS \"Specialty\",\n ((unnest(xpath('/ns:StoreSurvey/ns:SquareFeet/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"SquareFeet\",\n (unnest(xpath('/ns:StoreSurvey/ns:Brands/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(30) AS \"Brands\",\n (unnest(xpath('/ns:StoreSurvey/ns:Internet/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying(30) AS \"Internet\",\n ((unnest(xpath('/ns:StoreSurvey/ns:NumberEmployees/text()'::text, demographics, '{{ns,http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey}}'::text[])))::character varying)::integer AS \"NumberEmployees\"\n FROM sales.store;", "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/adventureworks/export-dir/schema/views/view.sql", "Suggestion": "", "GH": "", diff --git a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json index bbcf4613d..d02458c00 100644 --- a/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test-uqc/expectedAssessmentReport.json @@ -28,6 +28,12 @@ "TotalCount": 2, "InvalidCount": 0, "ObjectNames": "analytics.metrics, sales.orders" + }, + { + "ObjectType": "VIEW", + "TotalCount": 1, + "InvalidCount": 1, + "ObjectNames": "sales.employ_depart_view" } ] }, @@ -51,7 +57,18 @@ }, "UnsupportedDataTypes": null, "UnsupportedDataTypesDesc": "Data types of the source database that are not supported on the target YugabyteDB.", - "UnsupportedFeatures": null, + "UnsupportedFeatures": [ + { + "FeatureName": "Aggregate Functions", + "Objects": [ + { + "ObjectName": "sales.employ_depart_view", + "SqlStatement": "CREATE VIEW sales.employ_depart_view AS\n SELECT any_value(name) AS any_employee\n FROM public.employees;" + } + ], + "MinimumVersionsFixedIn": null + } + ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ { @@ -91,6 +108,12 @@ "Query": "SELECT metric_name, pg_advisory_lock(metric_id)\nFROM analytics.metrics\nWHERE metric_value \u003e $1", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", "MinimumVersionsFixedIn": null + }, + { + "ConstructTypeName": "Aggregate Functions", + "Query": "SELECT\n any_value(name) AS any_employee\n FROM employees", + "DocsLink": "", + "MinimumVersionsFixedIn": null } ], "UnsupportedPlPgSqlObjects": null diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 9ac91b514..ec45f22bf 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -489,11 +489,11 @@ "Objects": [ { "ObjectName": "public.sales_employees", - "SqlStatement": "CREATE VIEW public.sales_employees AS\n SELECT employees2.id,\n employees2.first_name,\n employees2.last_name,\n employees2.full_name\n FROM public.employees2\n WHERE ((employees2.department)::text = 'sales'::text)\n WITH CASCADED CHECK OPTION;" + "SqlStatement": "CREATE VIEW public.sales_employees AS\n SELECT id,\n first_name,\n last_name,\n full_name\n FROM public.employees2\n WHERE ((department)::text = 'sales'::text)\n WITH CASCADED CHECK OPTION;" }, { "ObjectName": "schema2.sales_employees", - "SqlStatement": "CREATE VIEW schema2.sales_employees AS\n SELECT employees2.id,\n employees2.first_name,\n employees2.last_name,\n employees2.full_name\n FROM schema2.employees2\n WHERE ((employees2.department)::text = 'sales'::text)\n WITH CASCADED CHECK OPTION;" + "SqlStatement": "CREATE VIEW schema2.sales_employees AS\n SELECT id,\n first_name,\n last_name,\n full_name\n FROM schema2.employees2\n WHERE ((department)::text = 'sales'::text)\n WITH CASCADED CHECK OPTION;" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#view-with-check-option-is-not-supported", @@ -594,7 +594,7 @@ "Objects": [ { "ObjectName": "public.ordersentry_view", - "SqlStatement": "CREATE VIEW public.ordersentry_view AS\n SELECT ordersentry.order_id,\n ordersentry.customer_name,\n ordersentry.product_name,\n ordersentry.quantity,\n ordersentry.price,\n XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name), XMLELEMENT(NAME \"Quantity\", ordersentry.quantity), XMLELEMENT(NAME \"TotalPrice\", (ordersentry.price * (ordersentry.quantity)::numeric))) AS order_xml,\n XMLCONCAT(XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name)) AS summary_xml,\n pg_try_advisory_lock((hashtext((ordersentry.customer_name || ordersentry.product_name)))::bigint) AS lock_acquired,\n ordersentry.ctid AS row_ctid,\n ordersentry.xmin AS transaction_id\n FROM public.ordersentry;" + "SqlStatement": "CREATE VIEW public.ordersentry_view AS\n SELECT order_id,\n customer_name,\n product_name,\n quantity,\n price,\n XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", customer_name), XMLELEMENT(NAME \"Product\", product_name), XMLELEMENT(NAME \"Quantity\", quantity), XMLELEMENT(NAME \"TotalPrice\", (price * (quantity)::numeric))) AS order_xml,\n XMLCONCAT(XMLELEMENT(NAME \"Customer\", customer_name), XMLELEMENT(NAME \"Product\", product_name)) AS summary_xml,\n pg_try_advisory_lock((hashtext((customer_name || product_name)))::bigint) AS lock_acquired,\n ctid AS row_ctid,\n xmin AS transaction_id\n FROM public.ordersentry;" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported", @@ -605,7 +605,7 @@ "Objects": [ { "ObjectName": "public.ordersentry_view", - "SqlStatement": "CREATE VIEW public.ordersentry_view AS\n SELECT ordersentry.order_id,\n ordersentry.customer_name,\n ordersentry.product_name,\n ordersentry.quantity,\n ordersentry.price,\n XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name), XMLELEMENT(NAME \"Quantity\", ordersentry.quantity), XMLELEMENT(NAME \"TotalPrice\", (ordersentry.price * (ordersentry.quantity)::numeric))) AS order_xml,\n XMLCONCAT(XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name)) AS summary_xml,\n pg_try_advisory_lock((hashtext((ordersentry.customer_name || ordersentry.product_name)))::bigint) AS lock_acquired,\n ordersentry.ctid AS row_ctid,\n ordersentry.xmin AS transaction_id\n FROM public.ordersentry;" + "SqlStatement": "CREATE VIEW public.ordersentry_view AS\n SELECT order_id,\n customer_name,\n product_name,\n quantity,\n price,\n XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", customer_name), XMLELEMENT(NAME \"Product\", product_name), XMLELEMENT(NAME \"Quantity\", quantity), XMLELEMENT(NAME \"TotalPrice\", (price * (quantity)::numeric))) AS order_xml,\n XMLCONCAT(XMLELEMENT(NAME \"Customer\", customer_name), XMLELEMENT(NAME \"Product\", product_name)) AS summary_xml,\n pg_try_advisory_lock((hashtext((customer_name || product_name)))::bigint) AS lock_acquired,\n ctid AS row_ctid,\n xmin AS transaction_id\n FROM public.ordersentry;" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", @@ -616,7 +616,7 @@ "Objects": [ { "ObjectName": "public.ordersentry_view", - "SqlStatement": "CREATE VIEW public.ordersentry_view AS\n SELECT ordersentry.order_id,\n ordersentry.customer_name,\n ordersentry.product_name,\n ordersentry.quantity,\n ordersentry.price,\n XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name), XMLELEMENT(NAME \"Quantity\", ordersentry.quantity), XMLELEMENT(NAME \"TotalPrice\", (ordersentry.price * (ordersentry.quantity)::numeric))) AS order_xml,\n XMLCONCAT(XMLELEMENT(NAME \"Customer\", ordersentry.customer_name), XMLELEMENT(NAME \"Product\", ordersentry.product_name)) AS summary_xml,\n pg_try_advisory_lock((hashtext((ordersentry.customer_name || ordersentry.product_name)))::bigint) AS lock_acquired,\n ordersentry.ctid AS row_ctid,\n ordersentry.xmin AS transaction_id\n FROM public.ordersentry;" + "SqlStatement": "CREATE VIEW public.ordersentry_view AS\n SELECT order_id,\n customer_name,\n product_name,\n quantity,\n price,\n XMLELEMENT(NAME \"OrderDetails\", XMLELEMENT(NAME \"Customer\", customer_name), XMLELEMENT(NAME \"Product\", product_name), XMLELEMENT(NAME \"Quantity\", quantity), XMLELEMENT(NAME \"TotalPrice\", (price * (quantity)::numeric))) AS order_xml,\n XMLCONCAT(XMLELEMENT(NAME \"Customer\", customer_name), XMLELEMENT(NAME \"Product\", product_name)) AS summary_xml,\n pg_try_advisory_lock((hashtext((customer_name || product_name)))::bigint) AS lock_acquired,\n ctid AS row_ctid,\n xmin AS transaction_id\n FROM public.ordersentry;" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", @@ -647,7 +647,7 @@ "Objects": [ { "ObjectName": "public.view_explicit_security_invoker", - "SqlStatement": "CREATE VIEW public.view_explicit_security_invoker WITH (security_invoker='true') AS\n SELECT employees.employee_id,\n employees.first_name\n FROM public.employees;" + "SqlStatement": "CREATE VIEW public.view_explicit_security_invoker WITH (security_invoker='true') AS\n SELECT employee_id,\n first_name\n FROM public.employees;" } ], "MinimumVersionsFixedIn": null diff --git a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json index dd49b1dc6..8cd48f94b 100755 --- a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json @@ -577,7 +577,7 @@ "Objects": [ { "ObjectName": "regress_rls_schema.bv1", - "SqlStatement": "CREATE VIEW regress_rls_schema.bv1 WITH (security_barrier='true') AS\n SELECT b1.a,\n b1.b\n FROM regress_rls_schema.b1\n WHERE (b1.a \u003e 0)\n WITH CASCADED CHECK OPTION;" + "SqlStatement": "CREATE VIEW regress_rls_schema.bv1 WITH (security_barrier='true') AS\n SELECT a,\n b\n FROM regress_rls_schema.b1\n WHERE (a \u003e 0)\n WITH CASCADED CHECK OPTION;" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#view-with-check-option-is-not-supported", diff --git a/migtests/tests/pg/omnibus/expected_files/expected_failed.sql b/migtests/tests/pg/omnibus/expected_files/expected_failed.sql index cc663a37c..ad72d6808 100755 --- a/migtests/tests/pg/omnibus/expected_files/expected_failed.sql +++ b/migtests/tests/pg/omnibus/expected_files/expected_failed.sql @@ -105,10 +105,10 @@ ERROR: VIEW WITH CASCADED CHECK OPTION not supported yet (SQLSTATE 0A000) File :/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/views/view.sql */ CREATE VIEW regress_rls_schema.bv1 WITH (security_barrier='true') AS - SELECT b1.a, - b1.b + SELECT a, + b FROM regress_rls_schema.b1 - WHERE (b1.a > 0) + WHERE (a > 0) WITH CASCADED CHECK OPTION; /* @@ -223,10 +223,10 @@ ERROR: relation "composite_type_examples.ordinary_table" does not exist (SQLSTAT File :/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/views/view.sql */ CREATE VIEW composite_type_examples.basic_view AS - SELECT ordinary_table.basic_, - ordinary_table._basic, - ordinary_table.nested, - ordinary_table._nested + SELECT basic_, + _basic, + nested, + _nested FROM composite_type_examples.ordinary_table; /* @@ -234,8 +234,8 @@ ERROR: relation "enum_example.bugs" does not exist (SQLSTATE 42P01) File :/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/views/view.sql */ CREATE VIEW enum_example._bugs AS - SELECT bugs.id, - bugs.status + SELECT id, + status FROM enum_example.bugs; /* @@ -243,11 +243,11 @@ ERROR: relation "foreign_db_example.technically_doesnt_exist" does not exist (SQ File :/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/views/view.sql */ CREATE VIEW public.foreign_db_example AS - SELECT technically_doesnt_exist.id, - technically_doesnt_exist.uses_type, - technically_doesnt_exist._uses_type, - technically_doesnt_exist.positive_number, - technically_doesnt_exist._positive_number + SELECT id, + uses_type, + _uses_type, + positive_number, + _positive_number FROM foreign_db_example.technically_doesnt_exist; /* @@ -255,7 +255,7 @@ ERROR: relation "range_type_example.example_tbl" does not exist (SQLSTATE 42P01) File :/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/views/view.sql */ CREATE VIEW range_type_example.depends_on_col_using_type AS - SELECT example_tbl.col + SELECT col FROM range_type_example.example_tbl; /* diff --git a/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json index c5a4f343e..a2eccb31e 100755 --- a/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json @@ -338,7 +338,7 @@ "ObjectType": "VIEW", "ObjectName": "regress_rls_schema.bv1", "Reason": "Schema containing VIEW WITH CHECK OPTION is not supported yet.", - "SqlStatement": "CREATE VIEW regress_rls_schema.bv1 WITH (security_barrier='true') AS\n SELECT b1.a,\n b1.b\n FROM regress_rls_schema.b1\n WHERE (b1.a \u003e 0)\n WITH CASCADED CHECK OPTION;", + "SqlStatement": "CREATE VIEW regress_rls_schema.bv1 WITH (security_barrier='true') AS\n SELECT a,\n b\n FROM regress_rls_schema.b1\n WHERE (a \u003e 0)\n WITH CASCADED CHECK OPTION;", "FilePath": "/home/ubuntu/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/views/view.sql", "Suggestion": "Use Trigger with INSTEAD OF clause on INSERT/UPDATE on view to get this functionality", "GH": "https://github.com/yugabyte/yugabyte-db/issues/22716", diff --git a/yb-voyager/cmd/constants.go b/yb-voyager/cmd/constants.go index b22891ae0..e825bc997 100644 --- a/yb-voyager/cmd/constants.go +++ b/yb-voyager/cmd/constants.go @@ -93,6 +93,9 @@ const ( CUTOVER_TO_SOURCE = "cutover-to-source" CUTOVER_TO_SOURCE_REPLICA = "cutover-to-source-replica" + CLIENT_MESSAGES_SESSION_VAR = "SET CLIENT_MIN_MESSAGES" + TRANSACTION_TIMEOUT_SESSION_VAR = "SET TRANSACTION_TIMEOUT" + // unsupported features of assess migration VIRTUAL_COLUMN = "VIRTUAL COLUMN" INHERITED_TYPE = "INHERITED TYPE" diff --git a/yb-voyager/cmd/importSchemaYugabyteDB.go b/yb-voyager/cmd/importSchemaYugabyteDB.go index 9cb37b96d..79720d147 100644 --- a/yb-voyager/cmd/importSchemaYugabyteDB.go +++ b/yb-voyager/cmd/importSchemaYugabyteDB.go @@ -86,7 +86,6 @@ func executeSqlFile(file string, objType string, skipFn func(string, string) boo }() sqlInfoArr := parseSqlFileForObjectType(file, objType) - var err error for _, sqlInfo := range sqlInfoArr { if conn == nil { conn = newTargetConn() @@ -97,17 +96,14 @@ func executeSqlFile(file string, objType string, skipFn func(string, string) boo if !setOrSelectStmt && skipFn != nil && skipFn(objType, sqlInfo.stmt) { continue } - - if objType == "TABLE" { - // Check if the statement should be skipped - skip, err := shouldSkipDDL(sqlInfo.stmt) - if err != nil { - return fmt.Errorf("error checking whether to skip DDL for statement [%s]: %v", sqlInfo.stmt, err) - } - if skip { - log.Infof("Skipping DDL: %s", sqlInfo.stmt) - continue - } + // Check if the statement should be skipped + skip, err := shouldSkipDDL(sqlInfo.stmt, objType) + if err != nil { + return fmt.Errorf("error checking whether to skip DDL for statement [%s]: %v", sqlInfo.stmt, err) + } + if skip { + log.Infof("Skipping DDL: %s", sqlInfo.stmt) + continue } err = executeSqlStmtWithRetries(&conn, sqlInfo, objType) @@ -118,14 +114,19 @@ func executeSqlFile(file string, objType string, skipFn func(string, string) boo return nil } -func shouldSkipDDL(stmt string) (bool, error) { +func shouldSkipDDL(stmt string, objType string) (bool, error) { stmt = strings.ToUpper(stmt) // pg_dump generate `SET client_min_messages = 'warning';`, but we want to get // NOTICE severity as well (which is the default), hence skipping this. - if strings.Contains(stmt, "SET CLIENT_MIN_MESSAGES") { + //pg_dump 17 gives this SET transaction_timeout = 0; + if strings.Contains(stmt, CLIENT_MESSAGES_SESSION_VAR) || strings.Contains(stmt, TRANSACTION_TIMEOUT_SESSION_VAR) { return true, nil } + if objType != TABLE { + return false, nil + } + skipReplicaIdentity := strings.Contains(stmt, "ALTER TABLE") && strings.Contains(stmt, "REPLICA IDENTITY") if skipReplicaIdentity { return true, nil From a85c6fcbfd0caf72cc271ab27ae4fc800326da01 Mon Sep 17 00:00:00 2001 From: Priyanshi Gupta Date: Fri, 27 Dec 2024 20:30:06 +0530 Subject: [PATCH 15/17] Fix: retrieving migration uuid early in the export schema phase (#2128) --- yb-voyager/cmd/exportSchema.go | 15 +++++++-------- yb-voyager/cmd/importData.go | 9 +++++---- yb-voyager/cmd/importDataFileCommand.go | 4 ++++ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/yb-voyager/cmd/exportSchema.go b/yb-voyager/cmd/exportSchema.go index 092a87c39..22de944ab 100644 --- a/yb-voyager/cmd/exportSchema.go +++ b/yb-voyager/cmd/exportSchema.go @@ -90,10 +90,15 @@ func exportSchema() error { utils.PrintAndLog("Schema is not exported yet. Ignoring --start-clean flag.\n\n") } CreateMigrationProjectIfNotExists(source.DBType, exportDir) - + err := retrieveMigrationUUID() + if err != nil { + log.Errorf("failed to get migration UUID: %v", err) + return fmt.Errorf("failed to get migration UUID: %w", err) + } + utils.PrintAndLog("export of schema for source type as '%s'\n", source.DBType) // Check connection with source database. - err := source.DB().Connect() + err = source.DB().Connect() if err != nil { log.Errorf("failed to connect to the source db: %s", err) return fmt.Errorf("failed to connect to the source db: %w", err) @@ -153,12 +158,6 @@ func exportSchema() error { } } - err = retrieveMigrationUUID() - if err != nil { - log.Errorf("failed to get migration UUID: %v", err) - return fmt.Errorf("failed to get migration UUID: %w", err) - } - exportSchemaStartEvent := createExportSchemaStartedEvent() controlPlane.ExportSchemaStarted(&exportSchemaStartEvent) diff --git a/yb-voyager/cmd/importData.go b/yb-voyager/cmd/importData.go index f70d48919..99e71c969 100644 --- a/yb-voyager/cmd/importData.go +++ b/yb-voyager/cmd/importData.go @@ -132,6 +132,11 @@ func importDataCommandFn(cmd *cobra.Command, args []string) { utils.ErrExit("Failed to get migration status record: %s", err) } + err = retrieveMigrationUUID() + if err != nil { + utils.ErrExit("failed to get migration UUID: %w", err) + } + // Check if target DB has the required permissions if tconf.RunGuardrailsChecks { checkImportDataPermissions() @@ -470,10 +475,6 @@ func updateTargetConfInMigrationStatus() { } func importData(importFileTasks []*ImportFileTask) { - err := retrieveMigrationUUID() - if err != nil { - utils.ErrExit("failed to get migration UUID: %w", err) - } if (importerRole == TARGET_DB_IMPORTER_ROLE || importerRole == IMPORT_FILE_ROLE) && (tconf.EnableUpsert) { if !utils.AskPrompt(color.RedString("WARNING: Ensure that tables on target YugabyteDB do not have secondary indexes. " + diff --git a/yb-voyager/cmd/importDataFileCommand.go b/yb-voyager/cmd/importDataFileCommand.go index 5985f4066..0deaee34b 100644 --- a/yb-voyager/cmd/importDataFileCommand.go +++ b/yb-voyager/cmd/importDataFileCommand.go @@ -91,6 +91,10 @@ var importDataFileCmd = &cobra.Command{ dataStore = datastore.NewDataStore(dataDir) importFileTasks := prepareImportFileTasks() prepareForImportDataCmd(importFileTasks) + err := retrieveMigrationUUID() + if err != nil { + utils.ErrExit("failed to get migration UUID: %w", err) + } importData(importFileTasks) packAndSendImportDataFilePayload(COMPLETE) From fa542b755d018ba7258a4f39e05a5fc7c1d45fee Mon Sep 17 00:00:00 2001 From: Aneesh Makala Date: Mon, 30 Dec 2024 10:15:24 +0530 Subject: [PATCH 16/17] FETCH .. WITH TIES issue detection (#2111) * Added new detector SelectStmtDetector --- .../dummy-export-dir/schema/tables/table.sql | 1 + .../dummy-export-dir/schema/views/view.sql | 7 +- .../tests/analyze-schema/expected_issues.json | 10 +++ migtests/tests/analyze-schema/summary.json | 11 ++-- .../expectedAssessmentReport.json | 64 ++++++++++++++++--- .../pg_assessment_report.sql | 14 ++++ yb-voyager/cmd/assessMigrationCommand.go | 1 + yb-voyager/cmd/constants.go | 1 + yb-voyager/go.mod | 3 - yb-voyager/go.sum | 5 -- yb-voyager/src/query/queryissue/constants.go | 4 +- yb-voyager/src/query/queryissue/detectors.go | 34 ++++++++++ yb-voyager/src/query/queryissue/issues_dml.go | 13 ++++ .../src/query/queryissue/issues_dml_test.go | 24 ++++++- .../query/queryissue/parser_issue_detector.go | 1 + .../queryissue/parser_issue_detector_test.go | 63 ++++++++++++++++++ .../src/query/queryparser/helpers_protomsg.go | 15 ++++- .../src/query/queryparser/helpers_struct.go | 4 ++ .../src/query/queryparser/traversal_proto.go | 1 + 19 files changed, 248 insertions(+), 28 deletions(-) diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql index 3d9cddc03..4da2e5f44 100755 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql @@ -386,6 +386,7 @@ CREATE TABLE public.locations ( CREATE TABLE image (title text, raster lo); +CREATE TABLE employees (id INT PRIMARY KEY, salary INT); -- create table with multirange data types -- Create tables with primary keys directly diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql index 157d311ce..1fc6a82cb 100644 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql @@ -34,6 +34,11 @@ CREATE VIEW public.orders_view AS orders.xmin AS transaction_id FROM public.orders; +CREATE VIEW top_employees_view AS SELECT * FROM ( + SELECT * FROM employees + ORDER BY salary DESC + FETCH FIRST 2 ROWS WITH TIES + ) AS top_employees; CREATE VIEW public.my_films_view AS SELECT jt.* FROM my_films, @@ -43,4 +48,4 @@ SELECT jt.* FROM kind text PATH '$.kind', NESTED PATH '$.films[*]' COLUMNS ( title text FORMAT JSON PATH '$.title' OMIT QUOTES, - director text PATH '$.director' KEEP QUOTES))) AS jt; \ No newline at end of file + director text PATH '$.director' KEEP QUOTES))) AS jt; diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index a6a63a62b..0bfd66b0e 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -1931,6 +1931,16 @@ "GH": "https://github.com/yugabyte/yugabyte-db/issues/25318", "MinimumVersionsFixedIn": null }, + { + "IssueType": "unsupported_features", + "ObjectType": "VIEW", + "ObjectName": "top_employees_view", + "Reason": "FETCH .. WITH TIES", + "SqlStatement": "CREATE VIEW top_employees_view AS SELECT * FROM (\n\t\t\tSELECT * FROM employees\n\t\t\tORDER BY salary DESC\n\t\t\tFETCH FIRST 2 ROWS WITH TIES\n\t\t) AS top_employees;", + "Suggestion": "No workaround available right now", + "GH": "", + "MinimumVersionsFixedIn": null + }, { "IssueType": "unsupported_datatypes", "ObjectType": "TABLE", diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index bdc832263..5e9353c7a 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -26,9 +26,10 @@ }, { "ObjectType": "TABLE", - "TotalCount": 57, + "TotalCount": 58, "InvalidCount": 49, - "ObjectNames": "image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table" }, + "ObjectNames": "employees, image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table" }, + { "ObjectType": "INDEX", "TotalCount": 43, @@ -50,9 +51,9 @@ }, { "ObjectType": "VIEW", - "TotalCount": 6, - "InvalidCount": 6, - "ObjectNames": "public.my_films_view, v1, v2, test, public.orders_view, view_name" + "TotalCount": 7, + "InvalidCount": 7, + "ObjectNames": "public.my_films_view, v1, v2, test, public.orders_view, view_name, top_employees_view" }, { "ObjectType": "TRIGGER", diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index ec45f22bf..76e5ffb54 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -38,15 +38,15 @@ }, { "ObjectType": "SEQUENCE", - "TotalCount": 41, + "TotalCount": 43, "InvalidCount": 0, - "ObjectNames": "public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.bigint_multirange_table_id_seq, public.date_multirange_table_id_seq, public.employees2_id_seq, public.employees3_id_seq, public.employees_employee_id_seq, public.ext_test_id_seq, public.int_multirange_table_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.numeric_multirange_table_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.timestamp_multirange_table_id_seq, public.timestamptz_multirange_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.bigint_multirange_table_id_seq, schema2.date_multirange_table_id_seq, schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.int_multirange_table_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.numeric_multirange_table_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.timestamp_multirange_table_id_seq, schema2.timestamptz_multirange_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" + "ObjectNames": "public.employeesforview_id_seq, schema2.employeesforview_id_seq, public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.bigint_multirange_table_id_seq, public.date_multirange_table_id_seq, public.employees2_id_seq, public.employees3_id_seq, public.employees_employee_id_seq, public.ext_test_id_seq, public.int_multirange_table_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.numeric_multirange_table_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.timestamp_multirange_table_id_seq, public.timestamptz_multirange_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.bigint_multirange_table_id_seq, schema2.date_multirange_table_id_seq, schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.int_multirange_table_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.numeric_multirange_table_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.timestamp_multirange_table_id_seq, schema2.timestamptz_multirange_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" }, { "ObjectType": "TABLE", - "TotalCount": 80, + "TotalCount": 82, "InvalidCount": 35, - "ObjectNames": "public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employees3, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" + "ObjectNames": "public.employeesforview, schema2.employeesforview, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employees3, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" }, { "ObjectType": "INDEX", @@ -73,9 +73,9 @@ }, { "ObjectType": "VIEW", - "TotalCount": 8, - "InvalidCount": 4, - "ObjectNames": "public.ordersentry_view, public.sales_employees, schema2.sales_employees, test_views.v1, test_views.v2, test_views.v3, test_views.v4, public.view_explicit_security_invoker" + "TotalCount": 10, + "InvalidCount": 6, + "ObjectNames": "public.ordersentry_view, public.sales_employees, schema2.sales_employees, test_views.v1, test_views.v2, test_views.v3, test_views.v4, public.view_explicit_security_invoker, schema2.top_employees_view, public.top_employees_view" }, { "ObjectType": "TRIGGER", @@ -162,6 +162,7 @@ "schema2.products", "schema2.foo", "schema2.Case_Sensitive_Columns", + "schema2.employeesforview", "schema2.with_example1", "test_views.xyz_mview", "test_views.view_table2", @@ -183,9 +184,10 @@ "schema2.int_multirange_table", "schema2.numeric_multirange_table", "schema2.timestamp_multirange_table", - "schema2.timestamptz_multirange_table" + "schema2.timestamptz_multirange_table", + "public.employeesforview" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 86 objects (78 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 88 objects (80 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": [ "public.combined_tbl", "public.citext_type", @@ -642,6 +644,20 @@ ], "MinimumVersionsFixedIn": null }, + { + "FeatureName": "FETCH .. WITH TIES Clause", + "Objects": [ + { + "ObjectName": "public.top_employees_view", + "SqlStatement": "CREATE VIEW public.top_employees_view AS\n SELECT id,\n first_name,\n last_name,\n salary\n FROM ( SELECT employeesforview.id,\n employeesforview.first_name,\n employeesforview.last_name,\n employeesforview.salary\n FROM public.employeesforview\n ORDER BY employeesforview.salary DESC\n FETCH FIRST 2 ROWS WITH TIES) top_employees;" + }, + { + "ObjectName": "schema2.top_employees_view", + "SqlStatement": "CREATE VIEW schema2.top_employees_view AS\n SELECT id,\n first_name,\n last_name,\n salary\n FROM ( SELECT employeesforview.id,\n employeesforview.first_name,\n employeesforview.last_name,\n employeesforview.salary\n FROM schema2.employeesforview\n ORDER BY employeesforview.salary DESC\n FETCH FIRST 2 ROWS WITH TIES) top_employees;" + } + ], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Security Invoker Views", "Objects": [ @@ -938,7 +954,7 @@ { "SchemaName": "public", "ObjectName": "employees", - "RowCount": 5, + "RowCount": 10, "ColumnCount": 4, "Reads": 0, "Writes": 10, @@ -2069,6 +2085,20 @@ "ParentTableName": "schema2.mixed_data_types_table1", "SizeInBytes": 8192 }, + { + "SchemaName": "public", + "ObjectName": "employeesforview", + "RowCount": 0, + "ColumnCount": 4, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, { "SchemaName": "public", "ObjectName": "employees3", @@ -2167,6 +2197,20 @@ "ParentTableName": null, "SizeInBytes": 0 }, + { + "SchemaName": "schema2", + "ObjectName": "employeesforview", + "RowCount": 0, + "ColumnCount": 4, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, { "SchemaName": "schema2", "ObjectName": "bigint_multirange_table", diff --git a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql index e1411914d..7e3c798ea 100644 --- a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql +++ b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql @@ -393,6 +393,20 @@ BEGIN END; $$ LANGUAGE plpgsql; +-- for FETCH .. WITH TIES +CREATE TABLE employeesForView ( + id SERIAL PRIMARY KEY, + first_name VARCHAR(50) NOT NULL, + last_name VARCHAR(50) NOT NULL, + salary NUMERIC(10, 2) NOT NULL +); + +CREATE VIEW top_employees_view AS SELECT * FROM ( + SELECT * FROM employeesForView + ORDER BY salary DESC + FETCH FIRST 2 ROWS WITH TIES + ) AS top_employees; + -- SECURITY INVOKER VIEW CREATE TABLE public.employees ( employee_id SERIAL PRIMARY KEY, diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 12cae929b..c54e998f5 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -1023,6 +1023,7 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SYSTEM_COLUMNS_NAME, queryissue.SYSTEM_COLUMNS_NAME, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.LARGE_OBJECT_FUNCTIONS_NAME, queryissue.LARGE_OBJECT_FUNCTIONS_NAME, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REGEX_FUNCTIONS_FEATURE, "", queryissue.REGEX_FUNCTIONS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(FETCH_WITH_TIES_FEATURE, "", queryissue.FETCH_WITH_TIES, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_QUERY_FUNCTIONS_NAME, "", queryissue.JSON_QUERY_FUNCTION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_CONSTRUCTOR_FUNCTION_NAME, "", queryissue.JSON_CONSTRUCTOR_FUNCTION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.AGGREGATION_FUNCTIONS_NAME, "", queryissue.AGGREGATE_FUNCTION, schemaAnalysisReport, false, "")) diff --git a/yb-voyager/cmd/constants.go b/yb-voyager/cmd/constants.go index e825bc997..a56ec1097 100644 --- a/yb-voyager/cmd/constants.go +++ b/yb-voyager/cmd/constants.go @@ -217,6 +217,7 @@ const ( BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE_FEATURE = "BEFORE ROW triggers on Partitioned tables" PK_UK_CONSTRAINT_ON_COMPLEX_DATATYPES_FEATURE = "Primary / Unique key constraints on complex datatypes" REGEX_FUNCTIONS_FEATURE = "Regex Functions" + FETCH_WITH_TIES_FEATURE = "FETCH .. WITH TIES Clause" // Migration caveats diff --git a/yb-voyager/go.mod b/yb-voyager/go.mod index 8cf7a703f..7df52aa40 100644 --- a/yb-voyager/go.mod +++ b/yb-voyager/go.mod @@ -15,7 +15,6 @@ require ( github.com/docker/go-connections v0.5.0 github.com/dustin/go-humanize v1.0.1 github.com/fatih/color v1.13.0 - github.com/fergusstrange/embedded-postgres v1.29.0 github.com/go-sql-driver/mysql v1.7.0 github.com/godror/godror v0.30.2 github.com/google/uuid v1.6.0 @@ -25,7 +24,6 @@ require ( github.com/jackc/pgconn v1.14.3 github.com/jackc/pgx/v4 v4.18.3 github.com/jackc/pgx/v5 v5.0.3 - github.com/lib/pq v1.10.9 github.com/mattn/go-sqlite3 v1.14.17 github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2 github.com/mitchellh/go-ps v1.0.0 @@ -83,7 +81,6 @@ require ( github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect - github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect diff --git a/yb-voyager/go.sum b/yb-voyager/go.sum index e68839fdc..1c06c76f9 100644 --- a/yb-voyager/go.sum +++ b/yb-voyager/go.sum @@ -919,8 +919,6 @@ github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fergusstrange/embedded-postgres v1.29.0 h1:Uv8hdhoiaNMuH0w8UuGXDHr60VoAQPFdgx7Qf3bzXJM= -github.com/fergusstrange/embedded-postgres v1.29.0/go.mod h1:t/MLs0h9ukYM6FSt99R7InCHs1nW0ordoVCcnzmpTYw= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -1971,8 +1969,6 @@ github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgk github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= -github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= -github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -2103,7 +2099,6 @@ go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index 329c25658..586957bf9 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -68,8 +68,8 @@ const ( ADVISORY_LOCKS_NAME = "Advisory Locks" SYSTEM_COLUMNS_NAME = "System Columns" XML_FUNCTIONS_NAME = "XML Functions" - - REGEX_FUNCTIONS = "REGEX_FUNCTIONS" + FETCH_WITH_TIES = "FETCH_WITH_TIES" + REGEX_FUNCTIONS = "REGEX_FUNCTIONS" MULTI_RANGE_DATATYPE = "MULTI_RANGE_DATATYPE" COPY_FROM_WHERE = "COPY FROM ... WHERE" diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index e6ec4c4cc..c834fa6db 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -206,6 +206,40 @@ func (d *RangeTableFuncDetector) GetIssues() []QueryIssue { return issues } +type SelectStmtDetector struct { + query string + limitOptionWithTiesDetected bool +} + +func NewSelectStmtDetector(query string) *SelectStmtDetector { + return &SelectStmtDetector{ + query: query, + } +} + +func (d *SelectStmtDetector) Detect(msg protoreflect.Message) error { + if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_SELECTSTMT_NODE { + selectStmtNode, err := queryparser.ProtoAsSelectStmt(msg) + if err != nil { + return err + } + // checks if a SelectStmt node uses a FETCH clause with TIES + // https://www.postgresql.org/docs/13/sql-select.html#SQL-LIMIT + if selectStmtNode.LimitOption == queryparser.LIMIT_OPTION_WITH_TIES { + d.limitOptionWithTiesDetected = true + } + } + return nil +} + +func (d *SelectStmtDetector) GetIssues() []QueryIssue { + var issues []QueryIssue + if d.limitOptionWithTiesDetected { + issues = append(issues, NewFetchWithTiesIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) + } + return issues +} + type CopyCommandUnsupportedConstructsDetector struct { query string copyFromWhereConstructDetected bool diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index e6915177c..7e6f8aed8 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -167,3 +167,16 @@ var copyOnErrorIssue = issue.Issue{ func NewCopyOnErrorIssue(objectType string, objectName string, sqlStatement string) QueryIssue { return newQueryIssue(copyOnErrorIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } + +var fetchWithTiesIssue = issue.Issue{ + Type: FETCH_WITH_TIES, + TypeName: "FETCH .. WITH TIES", + TypeDescription: "FETCH .. WITH TIES is not supported in YugabyteDB", + Suggestion: "No workaround available right now", + GH: "", + DocsLink: "", //TODO +} + +func NewFetchWithTiesIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(fetchWithTiesIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} diff --git a/yb-voyager/src/query/queryissue/issues_dml_test.go b/yb-voyager/src/query/queryissue/issues_dml_test.go index 8bbda2d34..cff5910c2 100644 --- a/yb-voyager/src/query/queryissue/issues_dml_test.go +++ b/yb-voyager/src/query/queryissue/issues_dml_test.go @@ -61,16 +61,35 @@ func testRegexFunctionsIssue(t *testing.T) { } } +func testFetchWithTiesIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + + stmts := []string{ + `SELECT * FROM employees + ORDER BY salary DESC + FETCH FIRST 2 ROWS WITH TIES;`, + } + + for _, stmt := range stmts { + _, err = conn.Exec(ctx, stmt) + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, `syntax error at or near "WITH"`, regexFunctionsIssue) + } +} + func testCopyOnErrorIssue(t *testing.T) { ctx := context.Background() conn, err := getConn() assert.NoError(t, err) defer conn.Close(context.Background()) + // In case the COPY ... ON_ERROR construct gets supported in the future, this test will fail with a different error message-something related to the data.csv file not being found. _, err = conn.Exec(ctx, `COPY pg_largeobject (loid, pageno, data) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE);`) assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "ERROR: option \"on_error\" not recognized (SQLSTATE 42601)", copyOnErrorIssue) - } func testCopyFromWhereIssue(t *testing.T) { @@ -171,6 +190,9 @@ func TestDMLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "regex functions", ybVersion), testRegexFunctionsIssue) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "fetch with ties", ybVersion), testFetchWithTiesIssue) + assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "copy on error", ybVersion), testCopyOnErrorIssue) assert.True(t, success) diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector.go b/yb-voyager/src/query/queryissue/parser_issue_detector.go index 9c32fca53..c7faf1209 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector.go @@ -375,6 +375,7 @@ func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) NewColumnRefDetector(query), NewXmlExprDetector(query), NewRangeTableFuncDetector(query), + NewSelectStmtDetector(query), NewCopyCommandUnsupportedConstructsDetector(query), NewJsonConstructorFuncDetector(query), NewJsonQueryFunctionDetector(query), diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index e4c6f09d1..e5b6315b0 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -653,6 +653,68 @@ func TestRegexFunctionsIssue(t *testing.T) { } +func TestFetchWithTiesInSelect(t *testing.T) { + + stmt1 := ` + SELECT * FROM employees + ORDER BY salary DESC + FETCH FIRST 2 ROWS WITH TIES;` + + // subquery + stmt2 := `SELECT * + FROM ( + SELECT * FROM employees + ORDER BY salary DESC + FETCH FIRST 2 ROWS WITH TIES + ) AS top_employees;` + + stmt3 := `CREATE VIEW top_employees_view AS + SELECT * + FROM ( + SELECT * FROM employees + ORDER BY salary DESC + FETCH FIRST 2 ROWS WITH TIES + ) AS top_employees;` + + expectedIssues := map[string][]QueryIssue{ + stmt1: []QueryIssue{NewFetchWithTiesIssue("DML_QUERY", "", stmt1)}, + stmt2: []QueryIssue{NewFetchWithTiesIssue("DML_QUERY", "", stmt2)}, + } + expectedDDLIssues := map[string][]QueryIssue{ + stmt3: []QueryIssue{NewFetchWithTiesIssue("VIEW", "top_employees_view", stmt3)}, + } + + parserIssueDetector := NewParserIssueDetector() + + for stmt, expectedIssues := range expectedIssues { + issues, err := parserIssueDetector.GetDMLIssues(stmt, ybversion.LatestStable) + + assert.NoError(t, err, "Error detecting issues for statement: %s", stmt) + + assert.Equal(t, len(expectedIssues), len(issues), "Mismatch in issue count for statement: %s", stmt) + for _, expectedIssue := range expectedIssues { + found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { + return cmp.Equal(expectedIssue, queryIssue) + }) + assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) + } + } + + for stmt, expectedIssues := range expectedDDLIssues { + issues, err := parserIssueDetector.GetDDLIssues(stmt, ybversion.LatestStable) + + assert.NoError(t, err, "Error detecting issues for statement: %s", stmt) + + assert.Equal(t, len(expectedIssues), len(issues), "Mismatch in issue count for statement: %s", stmt) + for _, expectedIssue := range expectedIssues { + found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { + return cmp.Equal(expectedIssue, queryIssue) + }) + assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) + } + } +} + func TestCopyUnsupportedConstructIssuesDetected(t *testing.T) { expectedIssues := map[string][]QueryIssue{ `COPY my_table FROM '/path/to/data.csv' WHERE col1 > 100;`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY my_table FROM '/path/to/data.csv' WHERE col1 > 100;`)}, @@ -683,6 +745,7 @@ func TestCopyUnsupportedConstructIssuesDetected(t *testing.T) { issues, err := parserIssueDetector.getDMLIssues(stmt) testutils.FatalIfError(t, err) assert.Equal(t, len(expectedIssues), len(issues)) + for _, expectedIssue := range expectedIssues { found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { return cmp.Equal(expectedIssue, queryIssue) diff --git a/yb-voyager/src/query/queryparser/helpers_protomsg.go b/yb-voyager/src/query/queryparser/helpers_protomsg.go index 8fe62bc82..de3204f3c 100644 --- a/yb-voyager/src/query/queryparser/helpers_protomsg.go +++ b/yb-voyager/src/query/queryparser/helpers_protomsg.go @@ -21,6 +21,7 @@ import ( pg_query "github.com/pganalyze/pg_query_go/v6" log "github.com/sirupsen/logrus" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" ) @@ -385,7 +386,7 @@ func GetListField(msg protoreflect.Message, fieldName string) protoreflect.List // GetEnumNumField retrieves a enum field from a message // FieldDescriptor{Syntax: proto3, FullName: pg_query.JsonFuncExpr.op, Number: 1, Cardinality: optional, Kind: enum, HasJSONName: true, JSONName: "op", Enum: pg_query.JsonExprOp} -//val:{json_func_expr:{op:JSON_QUERY_OP context_item:{raw_expr:{column_ref:{fields:{string:{sval:"details"}} location:2626}} format:{format_type:JS_FORMAT_DEFAULT encoding:JS_ENC_DEFAULT +// val:{json_func_expr:{op:JSON_QUERY_OP context_item:{raw_expr:{column_ref:{fields:{string:{sval:"details"}} location:2626}} format:{format_type:JS_FORMAT_DEFAULT encoding:JS_ENC_DEFAULT func GetEnumNumField(msg protoreflect.Message, fieldName string) protoreflect.EnumNumber { field := msg.Descriptor().Fields().ByName(protoreflect.Name(fieldName)) if field != nil && msg.Has(field) { @@ -407,6 +408,18 @@ func GetSchemaAndObjectName(nameList protoreflect.List) (string, string) { return schemaName, objectName } +func ProtoAsSelectStmt(msg protoreflect.Message) (*pg_query.SelectStmt, error) { + protoMsg, ok := msg.Interface().(proto.Message) + if !ok { + return nil, fmt.Errorf("failed to cast msg to proto.Message") + } + selectStmtNode, ok := protoMsg.(*pg_query.SelectStmt) + if !ok { + return nil, fmt.Errorf("failed to cast msg to %s", PG_QUERY_SELECTSTMT_NODE) + } + return selectStmtNode, nil +} + /* Example: options:{def_elem:{defname:"security_invoker" arg:{string:{sval:"true"}} defaction:DEFELEM_UNSPEC location:32}} diff --git a/yb-voyager/src/query/queryparser/helpers_struct.go b/yb-voyager/src/query/queryparser/helpers_struct.go index eafc90791..37d421bdf 100644 --- a/yb-voyager/src/query/queryparser/helpers_struct.go +++ b/yb-voyager/src/query/queryparser/helpers_struct.go @@ -23,6 +23,10 @@ import ( "github.com/samber/lo" ) +const ( + LIMIT_OPTION_WITH_TIES = pg_query.LimitOption_LIMIT_OPTION_WITH_TIES +) + func IsPLPGSQLObject(parseTree *pg_query.ParseResult) bool { // CREATE FUNCTION is same parser NODE for FUNCTION/PROCEDURE _, isPlPgSQLObject := getCreateFuncStmtNode(parseTree) diff --git a/yb-voyager/src/query/queryparser/traversal_proto.go b/yb-voyager/src/query/queryparser/traversal_proto.go index e742a8172..9ca3ae3ad 100644 --- a/yb-voyager/src/query/queryparser/traversal_proto.go +++ b/yb-voyager/src/query/queryparser/traversal_proto.go @@ -41,6 +41,7 @@ const ( PG_QUERY_INSERTSTMT_NODE = "pg_query.InsertStmt" PG_QUERY_UPDATESTMT_NODE = "pg_query.UpdateStmt" PG_QUERY_DELETESTMT_NODE = "pg_query.DeleteStmt" + PG_QUERY_SELECTSTMT_NODE = "pg_query.SelectStmt" PG_QUERY_JSON_OBJECT_AGG_NODE = "pg_query.JsonObjectAgg" PG_QUERY_JSON_ARRAY_AGG_NODE = "pg_query.JsonArrayAgg" From 29b09a8b9f6e961e0a7eb4bbd1a72ce217c3bd43 Mon Sep 17 00:00:00 2001 From: Shivansh Gahlot <42472145+ShivanshGahlot@users.noreply.github.com> Date: Fri, 3 Jan 2025 17:20:22 +0530 Subject: [PATCH 17/17] Shipping pg17 in the installer script and consequently in the docker installation too (#2133) --- .github/workflows/pg-13-migtests.yml | 10 ++++------ .github/workflows/pg-17-migtests.yml | 4 ++-- .github/workflows/pg-9-migtests.yml | 8 +++++--- installer_scripts/install-yb-voyager | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pg-13-migtests.yml b/.github/workflows/pg-13-migtests.yml index a64d351fa..d6b03ec16 100644 --- a/.github/workflows/pg-13-migtests.yml +++ b/.github/workflows/pg-13-migtests.yml @@ -70,10 +70,12 @@ jobs: cd installer_scripts yes | ./install-yb-voyager --install-from-local-source --only-pg-support sudo rm /usr/bin/pg_dump - sudo ln -s /usr/lib/postgresql/16/bin/pg_dump /usr/bin/pg_dump + sudo ln -s /usr/lib/postgresql/17/bin/pg_dump /usr/bin/pg_dump sudo rm /usr/bin/pg_restore - sudo ln -s /usr/lib/postgresql/16/bin/pg_restore /usr/bin/pg_restore + sudo ln -s /usr/lib/postgresql/17/bin/pg_restore /usr/bin/pg_restore pg_dump --version + pg_restore --version + psql --version env: ON_INSTALLER_ERROR_OUTPUT_LOG: Y @@ -119,10 +121,6 @@ jobs: if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test-export-data.sh pg/case-sensitivity-single-table - - name: "TEST: pg-dvdrental" - if: ${{ !cancelled() && matrix.test_group == 'offline' }} - run: migtests/scripts/run-test.sh pg/dvdrental - - name: "TEST: pg-datatypes" if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-test.sh pg/datatypes diff --git a/.github/workflows/pg-17-migtests.yml b/.github/workflows/pg-17-migtests.yml index 498a4e0cf..fc2aba70b 100644 --- a/.github/workflows/pg-17-migtests.yml +++ b/.github/workflows/pg-17-migtests.yml @@ -65,17 +65,17 @@ jobs: sudo apt install -y libpq-dev sudo apt install python3-psycopg2 - #TODO Remove the install PG 17 command once we do that in installer script - name: Run installer script to setup voyager run: | cd installer_scripts yes | ./install-yb-voyager --install-from-local-source --only-pg-support - sudo apt-get -y install postgresql-17 sudo rm /usr/bin/pg_dump sudo ln -s /usr/lib/postgresql/17/bin/pg_dump /usr/bin/pg_dump sudo rm /usr/bin/pg_restore sudo ln -s /usr/lib/postgresql/17/bin/pg_restore /usr/bin/pg_restore pg_dump --version + pg_restore --version + psql --version env: ON_INSTALLER_ERROR_OUTPUT_LOG: Y diff --git a/.github/workflows/pg-9-migtests.yml b/.github/workflows/pg-9-migtests.yml index ee406af1b..8fa3251d1 100644 --- a/.github/workflows/pg-9-migtests.yml +++ b/.github/workflows/pg-9-migtests.yml @@ -63,10 +63,12 @@ jobs: cd installer_scripts yes | ./install-yb-voyager --install-from-local-source --only-pg-support sudo rm /usr/bin/pg_dump - sudo ln -s /usr/lib/postgresql/16/bin/pg_dump /usr/bin/pg_dump + sudo ln -s /usr/lib/postgresql/17/bin/pg_dump /usr/bin/pg_dump sudo rm /usr/bin/pg_restore - sudo ln -s /usr/lib/postgresql/16/bin/pg_restore /usr/bin/pg_restore + sudo ln -s /usr/lib/postgresql/17/bin/pg_restore /usr/bin/pg_restore pg_dump --version + pg_restore --version + psql --version env: ON_INSTALLER_ERROR_OUTPUT_LOG: Y @@ -110,4 +112,4 @@ jobs: - name: "TEST: pg-constraints" if: ${{ !cancelled() }} - run: migtests/scripts/run-test.sh pg/constraints \ No newline at end of file + run: migtests/scripts/run-test.sh pg/constraints diff --git a/installer_scripts/install-yb-voyager b/installer_scripts/install-yb-voyager index 912f3e069..44a49c462 100755 --- a/installer_scripts/install-yb-voyager +++ b/installer_scripts/install-yb-voyager @@ -142,7 +142,7 @@ centos_main() { output "Installing RPM dependencies." $YUM_INSTALL which wget git gcc make 1>&2 $YUM_INSTALL https://download.postgresql.org/pub/repos/yum/reporpms/EL-${majorVersion}-x86_64/pgdg-redhat-repo-latest.noarch.rpm 1>&2 || true - $YUM_INSTALL postgresql16 1>&2 + $YUM_INSTALL postgresql17 1>&2 $YUM_INSTALL sqlite 1>&2 create_guardrail_scripts_dir create_pg_dump_args_file @@ -953,7 +953,7 @@ ubuntu_install_postgres() { sudo apt install -y postgresql-common 1>&2 echo | sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh 1>&2 sudo apt-get update 1>&2 - sudo apt-get -y install postgresql-16 1>&2 + sudo apt-get -y install postgresql-17 1>&2 output "Postgres Installed." }