From 579b0502c6f09d62bfcda4555d4a06eca46378b6 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Wed, 1 Jan 2025 21:28:59 +0000 Subject: [PATCH 01/16] Use Issue Type for issues lookup in schema analysis report - goal is to eventually get rid of using issue reason for lookups --- yb-voyager/cmd/assessMigrationCommand.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index f22eba98e..00fdfd2c4 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -1012,8 +1012,7 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem for _, indexMethod := range queryissue.UnsupportedIndexMethods { displayIndexMethod := strings.ToUpper(indexMethod) featureName := fmt.Sprintf("%s indexes", displayIndexMethod) - reason := fmt.Sprintf(INDEX_METHOD_ISSUE_REASON, displayIndexMethod) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(featureName, reason, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(featureName, "", queryissue.UNSUPPORTED_INDEX_METHOD, schemaAnalysisReport, false, "")) } unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CONSTRAINT_TRIGGERS_FEATURE, "", queryissue.CONSTRAINT_TRIGGER, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(INHERITED_TABLES_FEATURE, "", queryissue.INHERITANCE, schemaAnalysisReport, false, "")) @@ -1062,7 +1061,7 @@ func getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisReport utils.Schema unsupportedIndexDatatypes = append(unsupportedIndexDatatypes, "array") // adding it here only as we know issue form analyze will come with type unsupportedIndexDatatypes = append(unsupportedIndexDatatypes, "user_defined_type") // adding it here as we UDTs will come with this type. for _, unsupportedType := range unsupportedIndexDatatypes { - indexes := getUnsupportedFeaturesFromSchemaAnalysisReport(fmt.Sprintf("%s indexes", unsupportedType), fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, unsupportedType), "", schemaAnalysisReport, false, "") + indexes := getUnsupportedFeaturesFromSchemaAnalysisReport(fmt.Sprintf("%s indexes", unsupportedType), "", queryissue.INDEX_ON_COMPLEX_DATATYPE, schemaAnalysisReport, false, "") for _, object := range indexes.Objects { formattedObject := object formattedObject.ObjectName = fmt.Sprintf("%s: %s", strings.ToUpper(unsupportedType), object.ObjectName) From e2226a100cc4e0960184971f29020e099a5d7647 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Thu, 2 Jan 2025 19:28:49 +0000 Subject: [PATCH 02/16] reverting some cases --- yb-voyager/cmd/assessMigrationCommand.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 00fdfd2c4..f22eba98e 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -1012,7 +1012,8 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem for _, indexMethod := range queryissue.UnsupportedIndexMethods { displayIndexMethod := strings.ToUpper(indexMethod) featureName := fmt.Sprintf("%s indexes", displayIndexMethod) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(featureName, "", queryissue.UNSUPPORTED_INDEX_METHOD, schemaAnalysisReport, false, "")) + reason := fmt.Sprintf(INDEX_METHOD_ISSUE_REASON, displayIndexMethod) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(featureName, reason, "", schemaAnalysisReport, false, "")) } unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CONSTRAINT_TRIGGERS_FEATURE, "", queryissue.CONSTRAINT_TRIGGER, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(INHERITED_TABLES_FEATURE, "", queryissue.INHERITANCE, schemaAnalysisReport, false, "")) @@ -1061,7 +1062,7 @@ func getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisReport utils.Schema unsupportedIndexDatatypes = append(unsupportedIndexDatatypes, "array") // adding it here only as we know issue form analyze will come with type unsupportedIndexDatatypes = append(unsupportedIndexDatatypes, "user_defined_type") // adding it here as we UDTs will come with this type. for _, unsupportedType := range unsupportedIndexDatatypes { - indexes := getUnsupportedFeaturesFromSchemaAnalysisReport(fmt.Sprintf("%s indexes", unsupportedType), "", queryissue.INDEX_ON_COMPLEX_DATATYPE, schemaAnalysisReport, false, "") + indexes := getUnsupportedFeaturesFromSchemaAnalysisReport(fmt.Sprintf("%s indexes", unsupportedType), fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, unsupportedType), "", schemaAnalysisReport, false, "") for _, object := range indexes.Objects { formattedObject := object formattedObject.ObjectName = fmt.Sprintf("%s: %s", strings.ToUpper(unsupportedType), object.ObjectName) From 5bfc5eb566256c4272e40f02db365b83d7891631 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Mon, 6 Jan 2025 08:03:56 +0000 Subject: [PATCH 03/16] Refactor migration assessment codepath: flatten reported issues - Temporarily added new code to assessMigrationCommandV2.go - These functions will replace the existing codepath after UI updates are implemented. --- yb-voyager/cmd/analyzeSchema.go | 2 +- yb-voyager/cmd/assessMigrationCommand.go | 106 +++--- yb-voyager/cmd/assessMigrationCommandV2.go | 424 +++++++++++++++++++++ yb-voyager/cmd/constants.go | 68 ++-- yb-voyager/src/constants/constants.go | 9 + yb-voyager/src/issue/issue.go | 1 + yb-voyager/src/utils/commonVariables.go | 5 +- 7 files changed, 540 insertions(+), 75 deletions(-) create mode 100644 yb-voyager/cmd/assessMigrationCommandV2.go diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index f23c13d0b..07af301cf 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -623,7 +623,7 @@ func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fil issueType := UNSUPPORTED_FEATURES switch true { case isPlPgSQLIssue: - issueType = UNSUPPORTED_PLPGSQL_OBEJCTS + issueType = UNSUPPORTED_PLPGSQL_OBJECTS case slices.ContainsFunc(MigrationCaveatsIssues, func(i string) bool { //Adding the MIGRATION_CAVEATS issueType of the utils.Issue for these issueInstances in MigrationCaveatsIssues return strings.Contains(issueInstance.TypeName, i) diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index f22eba98e..43eb1feac 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -31,6 +31,7 @@ import ( "syscall" "text/template" + "github.com/davecgh/go-spew/spew" "github.com/fatih/color" "github.com/samber/lo" log "github.com/sirupsen/logrus" @@ -38,6 +39,7 @@ import ( "golang.org/x/exp/slices" "github.com/yugabyte/yb-voyager/yb-voyager/src/callhome" + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" "github.com/yugabyte/yb-voyager/yb-voyager/src/cp" "github.com/yugabyte/yb-voyager/yb-voyager/src/metadb" "github.com/yugabyte/yb-voyager/yb-voyager/src/migassessment" @@ -423,6 +425,15 @@ func assessMigration() (err error) { return fmt.Errorf("failed to generate assessment report: %w", err) } + assessmentIssues, err := fetchAllAssessmentIssues() + if err != nil { + log.Errorf("failed to fetch the assessment issues for migration complexity: %v", err) + } + utils.PrintAndLog("number of assessment issues detected: %d\n", len(assessmentIssues)) + for i, issue := range assessmentIssues { + fmt.Printf("AssessmentIssue[%d] = %s\n\n", i, spew.Sdump(issue)) + } + utils.PrintAndLog("Migration assessment completed successfully.") completedEvent := createMigrationAssessmentCompletedEvent() controlPlane.MigrationAssessmentCompleted(completedEvent) @@ -538,8 +549,8 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment } for _, unsupportedDataType := range ar.UnsupportedDataTypes { issues = append(issues, AssessmentIssueYugabyteD{ - Type: DATATYPE, - TypeDescription: DATATYPE_ISSUE_TYPE_DESCRIPTION, + Type: constants.DATATYPE, + TypeDescription: DATATYPE_CATEGORY_DESCRIPTION, Subtype: unsupportedDataType.DataType, ObjectName: fmt.Sprintf("%s.%s.%s", unsupportedDataType.SchemaName, unsupportedDataType.TableName, unsupportedDataType.ColumnName), SqlStatement: "", @@ -550,8 +561,8 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment for _, unsupportedFeature := range ar.UnsupportedFeatures { for _, object := range unsupportedFeature.Objects { issues = append(issues, AssessmentIssueYugabyteD{ - Type: FEATURE, - TypeDescription: FEATURE_ISSUE_TYPE_DESCRIPTION, + Type: constants.FEATURE, + TypeDescription: FEATURE_CATEGORY_DESCRIPTION, Subtype: unsupportedFeature.FeatureName, SubtypeDescription: unsupportedFeature.FeatureDescription, // TODO: test payload once we add desc for unsupported features ObjectName: object.ObjectName, @@ -565,8 +576,8 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment for _, migrationCaveat := range ar.MigrationCaveats { for _, object := range migrationCaveat.Objects { issues = append(issues, AssessmentIssueYugabyteD{ - Type: MIGRATION_CAVEATS, - TypeDescription: MIGRATION_CAVEATS_TYPE_DESCRIPTION, + Type: constants.MIGRATION_CAVEATS, + TypeDescription: MIGRATION_CAVEATS_CATEGORY_DESCRIPTION, Subtype: migrationCaveat.FeatureName, SubtypeDescription: migrationCaveat.FeatureDescription, ObjectName: object.ObjectName, @@ -579,8 +590,8 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment for _, uqc := range ar.UnsupportedQueryConstructs { issues = append(issues, AssessmentIssueYugabyteD{ - Type: QUERY_CONSTRUCT, - TypeDescription: UNSUPPORTED_QUERY_CONSTRUTS_DESCRIPTION, + Type: constants.QUERY_CONSTRUCT, + TypeDescription: UNSUPPORTED_QUERY_CONSTRUCTS_CATEGORY_DESCRIPTION, Subtype: uqc.ConstructTypeName, SqlStatement: uqc.Query, DocsLink: uqc.DocsLink, @@ -591,8 +602,8 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment for _, plpgsqlObjects := range ar.UnsupportedPlPgSqlObjects { for _, object := range plpgsqlObjects.Objects { issues = append(issues, AssessmentIssueYugabyteD{ - Type: PLPGSQL_OBJECT, - TypeDescription: UNSUPPPORTED_PLPGSQL_OBJECT_DESCRIPTION, + Type: constants.PLPGSQL_OBJECT, + TypeDescription: UNSUPPPORTED_PLPGSQL_OBJECT_CATEGORY_DESCRIPTION, Subtype: plpgsqlObjects.FeatureName, SubtypeDescription: plpgsqlObjects.FeatureDescription, ObjectName: object.ObjectName, @@ -888,7 +899,9 @@ func generateAssessmentReport() (err error) { return fmt.Errorf("failed to fetch columns with unsupported data types: %w", err) } assessmentReport.UnsupportedDataTypes = unsupportedDataTypes - assessmentReport.UnsupportedDataTypesDesc = DATATYPE_ISSUE_TYPE_DESCRIPTION + assessmentReport.UnsupportedDataTypesDesc = DATATYPE_CATEGORY_DESCRIPTION + + addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration, unsupportedDataTypesForLiveMigrationWithFForFB) assessmentReport.Sizing = migassessment.SizingReport assessmentReport.TableIndexStats, err = assessmentDB.FetchAllStats() @@ -897,7 +910,6 @@ func generateAssessmentReport() (err error) { } addNotesToAssessmentReport() - addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration, unsupportedDataTypesForLiveMigrationWithFForFB) postProcessingOfAssessmentReport() assessmentReportDir := filepath.Join(exportDir, "assessment", "reports") @@ -915,20 +927,16 @@ func generateAssessmentReport() (err error) { func getAssessmentReportContentFromAnalyzeSchema() error { /* - Here we are generating analyze schema report which converts issue instance to analyze schema issue - Then in assessment codepath we extract the required information from analyze schema issue which could have been done directly from issue instance(TODO) + Here we are generating analyze schema report which converts issue instance to analyze schema issue + Then in assessment codepath we extract the required information from analyze schema issue which could have been done directly from issue instance(TODO) - But current Limitation is analyze schema currently uses regexp etc to detect some issues(not using parser). + But current Limitation is analyze schema currently uses regexp etc to detect some issues(not using parser). */ schemaAnalysisReport := analyzeSchemaInternal(&source, true) assessmentReport.MigrationComplexity = schemaAnalysisReport.MigrationComplexity assessmentReport.SchemaSummary = schemaAnalysisReport.SchemaSummary - assessmentReport.SchemaSummary.Description = SCHEMA_SUMMARY_DESCRIPTION - if source.DBType == ORACLE { - assessmentReport.SchemaSummary.Description = SCHEMA_SUMMARY_DESCRIPTION_ORACLE - } + assessmentReport.SchemaSummary.Description = lo.Ternary(source.DBType == ORACLE, SCHEMA_SUMMARY_DESCRIPTION_ORACLE, SCHEMA_SUMMARY_DESCRIPTION) - // fetching unsupportedFeaturing with the help of Issues report in SchemaReport var unsupportedFeatures []UnsupportedFeature var err error switch source.DBType { @@ -940,13 +948,14 @@ func getAssessmentReportContentFromAnalyzeSchema() error { panic(fmt.Sprintf("unsupported source db type %q", source.DBType)) } if err != nil { - return fmt.Errorf("failed to fetch %s unsupported features: %w", source.DBType, err) + return fmt.Errorf("failed to fetch '%s' unsupported features: %w", source.DBType, err) } assessmentReport.UnsupportedFeatures = append(assessmentReport.UnsupportedFeatures, unsupportedFeatures...) - assessmentReport.UnsupportedFeaturesDesc = FEATURE_ISSUE_TYPE_DESCRIPTION + assessmentReport.UnsupportedFeaturesDesc = FEATURE_CATEGORY_DESCRIPTION + + // Ques: Do we still need this and REPORT_UNSUPPORTED_QUERY_CONSTRUCTS env var if utils.GetEnvAsBool("REPORT_UNSUPPORTED_PLPGSQL_OBJECTS", true) { - unsupportedPlpgSqlObjects := fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport) - assessmentReport.UnsupportedPlPgSqlObjects = unsupportedPlpgSqlObjects + assessmentReport.UnsupportedPlPgSqlObjects = fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport) } return nil } @@ -980,28 +989,30 @@ func getUnsupportedFeaturesFromSchemaAnalysisReport(featureName string, issueRea var minVersionsFixedIn map[string]*ybversion.YBVersion var minVersionsFixedInSet bool - for _, issue := range schemaAnalysisReport.Issues { - if !slices.Contains([]string{UNSUPPORTED_FEATURES, MIGRATION_CAVEATS}, issue.IssueType) { + for _, analyzeIssue := range schemaAnalysisReport.Issues { + if !slices.Contains([]string{UNSUPPORTED_FEATURES, MIGRATION_CAVEATS}, analyzeIssue.IssueType) { continue } - issueMatched := lo.Ternary[bool](issueType != "", issueType == issue.Type, strings.Contains(issue.Reason, issueReason)) + issueMatched := lo.Ternary[bool](issueType != "", issueType == analyzeIssue.Type, strings.Contains(analyzeIssue.Reason, issueReason)) if issueMatched { - objectInfo := ObjectInfo{ - ObjectName: issue.ObjectName, - SqlStatement: issue.SqlStatement, - } - link = issue.DocsLink - objects = append(objects, objectInfo) if !minVersionsFixedInSet { - minVersionsFixedIn = issue.MinimumVersionsFixedIn + minVersionsFixedIn = analyzeIssue.MinimumVersionsFixedIn minVersionsFixedInSet = true } - if !areMinVersionsFixedInEqual(minVersionsFixedIn, issue.MinimumVersionsFixedIn) { - utils.ErrExit("Issues belonging to UnsupportedFeature %s have different minimum versions fixed in: %v, %v", featureName, minVersionsFixedIn, issue.MinimumVersionsFixedIn) + if !areMinVersionsFixedInEqual(minVersionsFixedIn, analyzeIssue.MinimumVersionsFixedIn) { + utils.ErrExit("Issues belonging to UnsupportedFeature %s have different minimum versions fixed in: %v, %v", featureName, minVersionsFixedIn, analyzeIssue.MinimumVersionsFixedIn) } + + objectInfo := ObjectInfo{ + ObjectName: analyzeIssue.ObjectName, + SqlStatement: analyzeIssue.SqlStatement, + } + link = analyzeIssue.DocsLink + objects = append(objects, objectInfo) } } + return UnsupportedFeature{featureName, objects, displayDDLInHTML, link, description, minVersionsFixedIn} } @@ -1139,9 +1150,9 @@ func fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport utils.SchemaReport) []U if source.DBType != POSTGRESQL { return nil } - analyzeIssues := schemaAnalysisReport.Issues - plpgsqlIssues := lo.Filter(analyzeIssues, func(issue utils.AnalyzeSchemaIssue, _ int) bool { - return issue.IssueType == UNSUPPORTED_PLPGSQL_OBEJCTS + + plpgsqlIssues := lo.Filter(schemaAnalysisReport.Issues, func(issue utils.AnalyzeSchemaIssue, _ int) bool { + return issue.IssueType == UNSUPPORTED_PLPGSQL_OBJECTS }) groupPlpgsqlIssuesByReason := lo.GroupBy(plpgsqlIssues, func(issue utils.AnalyzeSchemaIssue) string { return issue.Reason @@ -1154,11 +1165,6 @@ func fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport utils.SchemaReport) []U var minVersionsFixedInSet bool for _, issue := range issues { - objects = append(objects, ObjectInfo{ - ObjectType: issue.ObjectType, - ObjectName: issue.ObjectName, - SqlStatement: issue.SqlStatement, - }) if !minVersionsFixedInSet { minVersionsFixedIn = issue.MinimumVersionsFixedIn minVersionsFixedInSet = true @@ -1166,6 +1172,12 @@ func fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport utils.SchemaReport) []U if !areMinVersionsFixedInEqual(minVersionsFixedIn, issue.MinimumVersionsFixedIn) { utils.ErrExit("Issues belonging to UnsupportedFeature %s have different minimum versions fixed in: %v, %v", reason, minVersionsFixedIn, issue.MinimumVersionsFixedIn) } + + objects = append(objects, ObjectInfo{ + ObjectType: issue.ObjectType, + ObjectName: issue.ObjectName, + SqlStatement: issue.SqlStatement, + }) docsLink = issue.DocsLink } feature := UnsupportedFeature{ @@ -1427,7 +1439,7 @@ func addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(FOREIGN_TABLE_CAVEAT_FEATURE, "", queryissue.FOREIGN_TABLE, schemaAnalysisReport, false, DESCRIPTION_FOREIGN_TABLES)) migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(POLICIES_CAVEAT_FEATURE, "", queryissue.POLICY_WITH_ROLES, - schemaAnalysisReport, false, DESCRIPTION_POLICY_ROLE_ISSUE)) + schemaAnalysisReport, false, DESCRIPTION_POLICY_ROLE_DESCRIPTION)) if len(unsupportedDataTypesForLiveMigration) > 0 { columns := make([]ObjectInfo, 0) @@ -1435,7 +1447,7 @@ func addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration columns = append(columns, ObjectInfo{ObjectName: fmt.Sprintf("%s.%s.%s (%s)", col.SchemaName, col.TableName, col.ColumnName, col.DataType)}) } if len(columns) > 0 { - migrationCaveats = append(migrationCaveats, UnsupportedFeature{UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE, columns, false, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_ISSUE, nil}) + migrationCaveats = append(migrationCaveats, UnsupportedFeature{UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE, columns, false, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_DESCRIPTION, nil}) } } if len(unsupportedDataTypesForLiveMigrationWithFForFB) > 0 { @@ -1444,7 +1456,7 @@ func addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration columns = append(columns, ObjectInfo{ObjectName: fmt.Sprintf("%s.%s.%s (%s)", col.SchemaName, col.TableName, col.ColumnName, col.DataType)}) } if len(columns) > 0 { - migrationCaveats = append(migrationCaveats, UnsupportedFeature{UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE, columns, false, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_ISSUE, nil}) + migrationCaveats = append(migrationCaveats, UnsupportedFeature{UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE, columns, false, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_DESCRIPTION, nil}) } } migrationCaveats = lo.Filter(migrationCaveats, func(m UnsupportedFeature, _ int) bool { diff --git a/yb-voyager/cmd/assessMigrationCommandV2.go b/yb-voyager/cmd/assessMigrationCommandV2.go new file mode 100644 index 000000000..2a7e21e67 --- /dev/null +++ b/yb-voyager/cmd/assessMigrationCommandV2.go @@ -0,0 +1,424 @@ +package cmd + +import ( + "fmt" + "strings" + + "github.com/samber/lo" + log "github.com/sirupsen/logrus" + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" + "github.com/yugabyte/yb-voyager/yb-voyager/src/migassessment" + "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryissue" + "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryparser" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" + "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" + "golang.org/x/exp/slices" +) + +// TODO: content of this function will move to generateAssessmentReport() +// For now this function will fetch all the issues which will be used by the migration complexity determination logic +func fetchAllAssessmentIssues() ([]AssessmentIssue, error) { + var assessmentIssues []AssessmentIssue + + assessmentIssues = append(assessmentIssues, getAssessmentReportContentFromAnalyzeSchemaV2()...) + + issues, err := fetchUnsupportedObjectTypesV2() + if err != nil { + return assessmentIssues, fmt.Errorf("failed to fetch unsupported object type issues: %w", err) + } + assessmentIssues = append(assessmentIssues, issues...) + + if utils.GetEnvAsBool("REPORT_UNSUPPORTED_QUERY_CONSTRUCTS", true) { + issues, err := fetchUnsupportedQueryConstructsV2() + if err != nil { + return assessmentIssues, fmt.Errorf("failed to fetch unsupported queries on YugabyteDB: %w", err) + } + assessmentIssues = append(assessmentIssues, issues...) + } + + unsupportedDataTypes, unsupportedDataTypesForLiveMigration, unsupportedDataTypesForLiveMigrationWithFForFB, err := fetchColumnsWithUnsupportedDataTypes() + if err != nil { + return assessmentIssues, fmt.Errorf("failed to fetch columns with unsupported data types: %w", err) + } + + assessmentIssues = append(assessmentIssues, getAssessmentIssuesForUnsupportedDatatypes(unsupportedDataTypes)...) + + assessmentIssues = append(assessmentIssues, fetchMigrationCaveatAssessmentIssues(unsupportedDataTypesForLiveMigration, unsupportedDataTypesForLiveMigrationWithFForFB)...) + + return assessmentIssues, nil +} + +// TODO: replacement of getAssessmentReportContentFromAnalyzeSchema() +func getAssessmentReportContentFromAnalyzeSchemaV2() []AssessmentIssue { + /* + Here we are generating analyze schema report which converts issue instance to analyze schema issue + Then in assessment codepath we extract the required information from analyze schema issue which could have been done directly from issue instance(TODO) + + But current Limitation is analyze schema currently uses regexp etc to detect some issues(not using parser). + */ + schemaAnalysisReport := analyzeSchemaInternal(&source, true) + assessmentReport.MigrationComplexity = schemaAnalysisReport.MigrationComplexity + assessmentReport.SchemaSummary = schemaAnalysisReport.SchemaSummary + assessmentReport.SchemaSummary.Description = lo.Ternary(source.DBType == ORACLE, SCHEMA_SUMMARY_DESCRIPTION_ORACLE, SCHEMA_SUMMARY_DESCRIPTION) + + var assessmentIssues []AssessmentIssue + switch source.DBType { + case ORACLE: + assessmentIssues = append(assessmentIssues, fetchUnsupportedOracleFeaturesFromSchemaReportV2(schemaAnalysisReport)...) + case POSTGRESQL: + assessmentIssues = append(assessmentIssues, fetchUnsupportedPGFeaturesFromSchemaReportV2(schemaAnalysisReport)...) + default: + panic(fmt.Sprintf("unsupported source db type %q", source.DBType)) + } + + // Ques: Do we still need this and REPORT_UNSUPPORTED_QUERY_CONSTRUCTS env var + if utils.GetEnvAsBool("REPORT_UNSUPPORTED_PLPGSQL_OBJECTS", true) { + assessmentIssues = append(assessmentIssues, fetchUnsupportedPlPgSQLObjectsV2(schemaAnalysisReport)...) + } + + return assessmentIssues +} + +// TODO: will replace fetchUnsupportedOracleFeaturesFromSchemaReport() +func fetchUnsupportedOracleFeaturesFromSchemaReportV2(schemaAnalysisReport utils.SchemaReport) []AssessmentIssue { + log.Infof("fetching assessment issues of feature category for Oracle...") + var assessmentIssues []AssessmentIssue + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, "", COMPOUND_TRIGGER_ISSUE_REASON, schemaAnalysisReport, "")...) + return assessmentIssues +} + +// TODO: will replace fetchUnsupportedPGFeaturesFromSchemaReport() +func fetchUnsupportedPGFeaturesFromSchemaReportV2(schemaAnalysisReport utils.SchemaReport) []AssessmentIssue { + log.Infof("fetching assessment issues of feature category for PG...") + + var assessmentIssues []AssessmentIssue + for _, indexMethod := range queryissue.UnsupportedIndexMethods { + displayIndexMethod := strings.ToUpper(indexMethod) + reason := fmt.Sprintf(INDEX_METHOD_ISSUE_REASON, displayIndexMethod) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, "", reason, schemaAnalysisReport, "")...) + } + + assessmentIssues = append(assessmentIssues, getIndexesOnComplexTypeUnsupportedFeatureV2(schemaAnalysisReport, queryissue.UnsupportedIndexDatatypes)...) + + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.CONSTRAINT_TRIGGER, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.INHERITANCE, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.STORED_GENERATED_COLUMNS, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, "", CONVERSION_ISSUE_REASON, schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.MULTI_COLUMN_GIN_INDEX, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.ALTER_TABLE_SET_COLUMN_ATTRIBUTE, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.ALTER_TABLE_DISABLE_RULE, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.ALTER_TABLE_CLUSTER_ON, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.STORAGE_PARAMETER, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, "", UNSUPPORTED_EXTENSION_ISSUE, schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.EXCLUSION_CONSTRAINTS, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.DEFERRABLE_CONSTRAINTS, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, "", VIEW_CHECK_OPTION_ISSUE, schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.PK_UK_ON_COMPLEX_DATATYPE, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.UNLOGGED_TABLE, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.REFERENCING_CLAUSE_IN_TRIGGER, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.BEFORE_ROW_TRIGGER_ON_PARTITIONED_TABLE, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.ADVISORY_LOCKS, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.XML_FUNCTIONS, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.SYSTEM_COLUMNS, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.LARGE_OBJECT_FUNCTIONS, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.REGEX_FUNCTIONS, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.FETCH_WITH_TIES, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.JSON_QUERY_FUNCTION, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.JSON_CONSTRUCTOR_FUNCTION, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.AGGREGATE_FUNCTION, "", schemaAnalysisReport, "")...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.SECURITY_INVOKER_VIEWS, "", schemaAnalysisReport, "")...) + + return assessmentIssues +} + +func getIndexesOnComplexTypeUnsupportedFeatureV2(schemaAnalysisReport utils.SchemaReport, unsupportedIndexDatatypes []string) []AssessmentIssue { + var assessmentIssues []AssessmentIssue + // TODO: include MinimumVersionsFixedIn + + unsupportedIndexDatatypes = append(unsupportedIndexDatatypes, "array") // adding it here only as we know issue form analyze will come with type + unsupportedIndexDatatypes = append(unsupportedIndexDatatypes, "user_defined_type") // adding it here as we UDTs will come with this type. + for _, unsupportedType := range unsupportedIndexDatatypes { + // formattedObject.ObjectName = fmt.Sprintf("%s: %s", strings.ToUpper(unsupportedType), object.ObjectName) + issueReason := fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, unsupportedType) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, "", issueReason, schemaAnalysisReport, "")...) + } + return assessmentIssues +} + +// TODO: replacement of fetchUnsupportedObjectTypes() +func fetchUnsupportedObjectTypesV2() ([]AssessmentIssue, error) { + if source.DBType != ORACLE { + return nil, nil + } + + query := fmt.Sprintf(`SELECT schema_name, object_name, object_type FROM %s`, migassessment.OBJECT_TYPE_MAPPING) + rows, err := assessmentDB.Query(query) + if err != nil { + return nil, fmt.Errorf("error querying-%s: %w", query, err) + } + defer func() { + closeErr := rows.Close() + if closeErr != nil { + log.Warnf("error closing rows while fetching object type mapping metadata: %v", err) + } + }() + + var assessmentIssues []AssessmentIssue + for rows.Next() { + var schemaName, objectName, objectType string + err = rows.Scan(&schemaName, &objectName, &objectType) + if err != nil { + return nil, fmt.Errorf("error scanning rows:%w", err) + } + + switch { + case slices.Contains(OracleUnsupportedIndexTypes, objectType): + assessmentIssues = append(assessmentIssues, AssessmentIssue{ + Category: constants.FEATURE, + TypeName: UNSUPPORTED_INDEXES_FEATURE, + Impact: "", // TODO + ObjectType: "INDEX", + ObjectName: fmt.Sprintf("Index Name: %s, Index Type=%s", objectName, objectType), + }) + case objectType == VIRTUAL_COLUMN: + assessmentIssues = append(assessmentIssues, AssessmentIssue{ + Category: constants.FEATURE, + TypeName: VIRTUAL_COLUMNS_FEATURE, + ObjectName: objectName, + }) + case objectType == INHERITED_TYPE: + assessmentIssues = append(assessmentIssues, AssessmentIssue{ + Category: constants.FEATURE, + TypeName: INHERITED_TYPES_FEATURE, + ObjectName: objectName, + }) + case objectType == REFERENCE_PARTITION || objectType == SYSTEM_PARTITION: + referenceOrTablePartitionPresent = true + assessmentIssues = append(assessmentIssues, AssessmentIssue{ + Category: constants.FEATURE, + TypeName: UNSUPPORTED_PARTITIONING_METHODS_FEATURE, + ObjectType: "TABLE", + ObjectName: fmt.Sprintf("Table Name: %s, Partition Method: %s", objectName, objectType), + }) + } + } + + return assessmentIssues, nil +} + +// // TODO: replacement of fetchUnsupportedPlPgSQLObjects() +func fetchUnsupportedPlPgSQLObjectsV2(schemaAnalysisReport utils.SchemaReport) []AssessmentIssue { + if source.DBType != POSTGRESQL { + return nil + } + + plpgsqlIssues := lo.Filter(schemaAnalysisReport.Issues, func(issue utils.AnalyzeSchemaIssue, _ int) bool { + return issue.IssueType == UNSUPPORTED_PLPGSQL_OBJECTS + }) + groupedPlpgsqlIssuesByReason := lo.GroupBy(plpgsqlIssues, func(issue utils.AnalyzeSchemaIssue) string { + return issue.Reason + }) + + var assessmentIssues []AssessmentIssue + for reason, issues := range groupedPlpgsqlIssuesByReason { + var minVersionsFixedIn map[string]*ybversion.YBVersion + var minVersionsFixedInSet bool + + for _, issue := range issues { + if !minVersionsFixedInSet { + minVersionsFixedIn = issue.MinimumVersionsFixedIn + minVersionsFixedInSet = true + } + if !areMinVersionsFixedInEqual(minVersionsFixedIn, issue.MinimumVersionsFixedIn) { + utils.ErrExit("Issues belonging to UnsupportedFeature %s have different minimum versions fixed in: %v, %v", reason, minVersionsFixedIn, issue.MinimumVersionsFixedIn) + } + + assessmentIssues = append(assessmentIssues, AssessmentIssue{ + Category: constants.PLPGSQL_OBJECT, + TypeName: reason, + Impact: issue.Impact, // TODO + ObjectType: issue.ObjectType, + ObjectName: issue.ObjectName, + SqlStatement: issue.SqlStatement, + DocsLink: issue.DocsLink, + MinimumVersionFixedIn: issue.MinimumVersionsFixedIn, + }) + } + } + + return assessmentIssues +} + +// Q: do we no need of displayDDLInHTML in this approach? DDL can always be there for issues in the table. +func getAssessmentIssuesFromSchemaAnalysisReport(category string, issueType string, issueReason string, schemaAnalysisReport utils.SchemaReport, issueDescription string) []AssessmentIssue { + log.Infof("filtering issues for type: %s", issueType) + var issues []AssessmentIssue + var minVersionsFixedIn map[string]*ybversion.YBVersion + var minVersionsFixedInSet bool + for _, analyzeIssue := range schemaAnalysisReport.Issues { + if !slices.Contains([]string{UNSUPPORTED_FEATURES, MIGRATION_CAVEATS}, analyzeIssue.IssueType) { + continue + } + + issueMatched := lo.Ternary[bool](issueType != "", issueType == analyzeIssue.Type, strings.Contains(analyzeIssue.Reason, issueReason)) + if issueMatched { + if !minVersionsFixedInSet { + minVersionsFixedIn = analyzeIssue.MinimumVersionsFixedIn + minVersionsFixedInSet = true + } + if !areMinVersionsFixedInEqual(minVersionsFixedIn, analyzeIssue.MinimumVersionsFixedIn) { + utils.ErrExit("Issues belonging to type %s have different minimum versions fixed in: %v, %v", analyzeIssue.Type, minVersionsFixedIn, analyzeIssue.MinimumVersionsFixedIn) + } + + issues = append(issues, AssessmentIssue{ + Category: category, + CategoryDescription: GetCategoryDescription(category), + TypeName: analyzeIssue.Type, + TypeDescription: issueDescription, // TODO: verify + Impact: analyzeIssue.Impact, + ObjectType: analyzeIssue.ObjectType, + ObjectName: analyzeIssue.ObjectName, + SqlStatement: analyzeIssue.SqlStatement, + DocsLink: analyzeIssue.DocsLink, + MinimumVersionFixedIn: minVersionsFixedIn, + }) + } + } + + return issues +} + +// TODO: soon to replace fetchUnsupportedQueryConstructs() +func fetchUnsupportedQueryConstructsV2() ([]AssessmentIssue, error) { + if source.DBType != POSTGRESQL { + return nil, nil + } + + query := fmt.Sprintf("SELECT DISTINCT query from %s", migassessment.DB_QUERIES_SUMMARY) + rows, err := assessmentDB.Query(query) + if err != nil { + return nil, fmt.Errorf("error querying=%s on assessmentDB: %w", query, err) + } + defer func() { + closeErr := rows.Close() + if closeErr != nil { + log.Warnf("error closing rows while fetching database queries summary metadata: %v", err) + } + }() + + var executedQueries []string + for rows.Next() { + var executedQuery string + err := rows.Scan(&executedQuery) + if err != nil { + return nil, fmt.Errorf("error scanning rows: %w", err) + } + executedQueries = append(executedQueries, executedQuery) + } + + if len(executedQueries) == 0 { + log.Infof("queries info not present in the assessment metadata for detecting unsupported query constructs") + return nil, nil + } + + var assessmentIssues []AssessmentIssue + for i := 0; i < len(executedQueries); i++ { + query := executedQueries[i] + log.Debugf("fetching unsupported query constructs for query - [%s]", query) + collectedSchemaList, err := queryparser.GetSchemaUsed(query) + if err != nil { // no need to error out if failed to get schemas for a query + log.Errorf("failed to get schemas used for query [%s]: %v", query, err) + continue + } + + log.Infof("collected schema list %v(len=%d) for query [%s]", collectedSchemaList, len(collectedSchemaList), query) + if !considerQueryForIssueDetection(collectedSchemaList) { + log.Infof("ignoring query due to difference in collected schema list %v(len=%d) vs source schema list %v(len=%d)", + collectedSchemaList, len(collectedSchemaList), source.GetSchemaList(), len(source.GetSchemaList())) + continue + } + + issues, err := parserIssueDetector.GetDMLIssues(query, targetDbVersion) + if err != nil { + log.Errorf("failed while trying to fetch query issues in query - [%s]: %v", + query, err) + } + + for _, issue := range issues { + issue := AssessmentIssue{ + Category: constants.QUERY_CONSTRUCT, + TypeName: issue.TypeName, + SqlStatement: issue.SqlStatement, + DocsLink: issue.DocsLink, + MinimumVersionFixedIn: issue.MinimumVersionsFixedIn, + } + assessmentIssues = append(assessmentIssues, issue) + } + } + + return assessmentIssues, nil +} + +func getAssessmentIssuesForUnsupportedDatatypes(unsupportedDatatypes []utils.TableColumnsDataTypes) []AssessmentIssue { + var assessmentIssues []AssessmentIssue + for _, colInfo := range unsupportedDatatypes { + qualifiedColName := fmt.Sprintf("%s.%s.%s", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName) + issue := AssessmentIssue{ + Category: constants.DATATYPE, + CategoryDescription: GetCategoryDescription(constants.DATATYPE), + TypeName: colInfo.DataType, // TODO: maybe name it like "unsupported datatype - geometry" + Impact: "", // TODO + ObjectType: constants.COLUMN, + ObjectName: qualifiedColName, + DocsLink: "", // TODO + MinimumVersionFixedIn: nil, // TODO + } + assessmentIssues = append(assessmentIssues, issue) + } + + return assessmentIssues +} + +// TODO: soon to replace addMigrationCaveatsToAssessmentReport() +func fetchMigrationCaveatAssessmentIssues(unsupportedDataTypesForLiveMigration []utils.TableColumnsDataTypes, unsupportedDataTypesForLiveMigrationWithFForFB []utils.TableColumnsDataTypes) []AssessmentIssue { + var assessmentIssues []AssessmentIssue + switch source.DBType { + case POSTGRESQL: + log.Infof("fetching migration caveat category assessment issues") + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.MIGRATION_CAVEATS, queryissue.ALTER_TABLE_ADD_PK_ON_PARTITIONED_TABLE, "", schemaAnalysisReport, DESCRIPTION_ADD_PK_TO_PARTITION_TABLE)...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.MIGRATION_CAVEATS, queryissue.FOREIGN_TABLE, "", schemaAnalysisReport, DESCRIPTION_FOREIGN_TABLES)...) + assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.MIGRATION_CAVEATS, queryissue.POLICY_WITH_ROLES, "", schemaAnalysisReport, DESCRIPTION_POLICY_ROLE_DESCRIPTION)...) + + if len(unsupportedDataTypesForLiveMigration) > 0 { + for _, colInfo := range unsupportedDataTypesForLiveMigration { + issue := AssessmentIssue{ + Category: constants.MIGRATION_CAVEATS, + CategoryDescription: "", // TODO + TypeName: UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE, // TODO add object type in type name + TypeDescription: UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_DESCRIPTION, + ObjectType: constants.COLUMN, + ObjectName: fmt.Sprintf("%s.%s.%s", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName), + DocsLink: UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, + } + assessmentIssues = append(assessmentIssues, issue) + } + } + + if len(unsupportedDataTypesForLiveMigrationWithFForFB) > 0 { + for _, colInfo := range unsupportedDataTypesForLiveMigrationWithFForFB { + issue := AssessmentIssue{ + Category: constants.MIGRATION_CAVEATS, + CategoryDescription: "", // TODO + TypeName: UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE, // TODO add object type in type name + TypeDescription: UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_DESCRIPTION, + ObjectType: constants.COLUMN, + ObjectName: fmt.Sprintf("%s.%s.%s", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName), + DocsLink: UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, + } + assessmentIssues = append(assessmentIssues, issue) + } + } + } + + return assessmentIssues +} diff --git a/yb-voyager/cmd/constants.go b/yb-voyager/cmd/constants.go index f4d9cc4fc..e1f0ff1a6 100644 --- a/yb-voyager/cmd/constants.go +++ b/yb-voyager/cmd/constants.go @@ -15,6 +15,11 @@ limitations under the License. */ package cmd +import ( + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" +) + const ( KB = 1024 MB = 1024 * 1024 @@ -104,7 +109,8 @@ const ( UNSUPPORTED_FEATURES = "unsupported_features" UNSUPPORTED_DATATYPES = "unsupported_datatypes" - UNSUPPORTED_PLPGSQL_OBEJCTS = "unsupported_plpgsql_objects" + UNSUPPORTED_PLPGSQL_OBJECTS = "unsupported_plpgsql_objects" + MIGRATION_CAVEATS = "migration_caveats" REPORT_UNSUPPORTED_QUERY_CONSTRUCTS = "REPORT_UNSUPPORTED_QUERY_CONSTRUCTS" HTML = "html" @@ -173,21 +179,14 @@ const ( List of all the features we are reporting as part of Unsupported features and Migration caveats */ const ( - // AssessmentIssue types used in YugabyteD payload - FEATURE = "feature" - DATATYPE = "datatype" - QUERY_CONSTRUCT = "query_construct" // confused: in json for some values we are using space separated and for some snake_case - MIGRATION_CAVEATS = "migration_caveats" - PLPGSQL_OBJECT = "plpgsql_object" - // Description - FEATURE_ISSUE_TYPE_DESCRIPTION = "Features of the source database that are not supported on the target YugabyteDB." - DATATYPE_ISSUE_TYPE_DESCRIPTION = "Data types of the source database that are not supported on the target YugabyteDB." - MIGRATION_CAVEATS_TYPE_DESCRIPTION = "Migration Caveats highlights the current limitations with the migration workflow." - UNSUPPORTED_QUERY_CONSTRUTS_DESCRIPTION = "Source database queries not supported in YugabyteDB, identified by scanning system tables." - UNSUPPPORTED_PLPGSQL_OBJECT_DESCRIPTION = "Source schema objects having unsupported statements on the target YugabyteDB in PL/pgSQL code block" - SCHEMA_SUMMARY_DESCRIPTION = "Objects that will be created on the target YugabyteDB." - SCHEMA_SUMMARY_DESCRIPTION_ORACLE = SCHEMA_SUMMARY_DESCRIPTION + " Some of the index and sequence names might be different from those in the source database." + FEATURE_CATEGORY_DESCRIPTION = "Features of the source database that are not supported on the target YugabyteDB." + DATATYPE_CATEGORY_DESCRIPTION = "Data types of the source database that are not supported on the target YugabyteDB." + MIGRATION_CAVEATS_CATEGORY_DESCRIPTION = "Migration Caveats highlights the current limitations with the migration workflow." + UNSUPPORTED_QUERY_CONSTRUCTS_CATEGORY_DESCRIPTION = "Source database queries not supported in YugabyteDB, identified by scanning system tables." + UNSUPPPORTED_PLPGSQL_OBJECT_CATEGORY_DESCRIPTION = "Source schema objects having unsupported statements on the target YugabyteDB in PL/pgSQL code block" + SCHEMA_SUMMARY_DESCRIPTION = "Objects that will be created on the target YugabyteDB." + SCHEMA_SUMMARY_DESCRIPTION_ORACLE = SCHEMA_SUMMARY_DESCRIPTION + " Some of the index and sequence names might be different from those in the source database." //Unsupported Features @@ -222,16 +221,16 @@ const ( // Migration caveats //POSTGRESQL - ALTER_PARTITION_ADD_PK_CAVEAT_FEATURE = "Alter partitioned tables to add Primary Key" - FOREIGN_TABLE_CAVEAT_FEATURE = "Foreign tables" - POLICIES_CAVEAT_FEATURE = "Policies" - UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE = "Unsupported Data Types for Live Migration" - UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE = "Unsupported Data Types for Live Migration with Fall-forward/Fallback" - UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_ISSUE = "There are some data types in the schema that are not supported by live migration of data. These columns will be excluded when exporting and importing data in live migration workflows." - UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_ISSUE = "There are some data types in the schema that are not supported by live migration with fall-forward/fall-back. These columns will be excluded when exporting and importing data in live migration workflows." - DESCRIPTION_ADD_PK_TO_PARTITION_TABLE = `After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.` - DESCRIPTION_FOREIGN_TABLES = `During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.` - DESCRIPTION_POLICY_ROLE_ISSUE = `There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.` + ALTER_PARTITION_ADD_PK_CAVEAT_FEATURE = "Alter partitioned tables to add Primary Key" + FOREIGN_TABLE_CAVEAT_FEATURE = "Foreign tables" + POLICIES_CAVEAT_FEATURE = "Policies" + UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE = "Unsupported Data Types for Live Migration" + UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE = "Unsupported Data Types for Live Migration with Fall-forward/Fallback" + UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_DESCRIPTION = "There are some data types in the schema that are not supported by live migration of data. These columns will be excluded when exporting and importing data in live migration workflows." + UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_DESCRIPTION = "There are some data types in the schema that are not supported by live migration with fall-forward/fall-back. These columns will be excluded when exporting and importing data in live migration workflows." + DESCRIPTION_ADD_PK_TO_PARTITION_TABLE = `After export schema, the ALTER table should be merged with CREATE table for partitioned tables as alter of partitioned tables to add primary key is not supported.` + DESCRIPTION_FOREIGN_TABLES = `During the export schema phase, SERVER and USER MAPPING objects are not exported. These should be manually created to make the foreign tables work.` + DESCRIPTION_POLICY_ROLE_DESCRIPTION = `There are some policies that are created for certain users/roles. During the export schema phase, USERs and GRANTs are not exported. Therefore, they will have to be manually created before running import schema.` ) var supportedSourceDBTypes = []string{ORACLE, MYSQL, POSTGRESQL, YUGABYTEDB} @@ -244,3 +243,22 @@ var validSSLModes = map[string][]string{ } var EVENT_BATCH_MAX_RETRY_COUNT = 50 + +// returns the description for a given assessment issue category +func GetCategoryDescription(category string) string { + switch category { + case constants.FEATURE: + return FEATURE_CATEGORY_DESCRIPTION + case constants.DATATYPE: + return DATATYPE_CATEGORY_DESCRIPTION + case constants.QUERY_CONSTRUCT: + return UNSUPPORTED_QUERY_CONSTRUCTS_CATEGORY_DESCRIPTION + case constants.PLPGSQL_OBJECT: + return UNSUPPPORTED_PLPGSQL_OBJECT_CATEGORY_DESCRIPTION + case constants.MIGRATION_CAVEATS: + return MIGRATION_CAVEATS_CATEGORY_DESCRIPTION + default: + utils.ErrExit("unsupported assessment issue category %q", category) + } + return "" +} diff --git a/yb-voyager/src/constants/constants.go b/yb-voyager/src/constants/constants.go index 7912fb1d5..59710a9e4 100644 --- a/yb-voyager/src/constants/constants.go +++ b/yb-voyager/src/constants/constants.go @@ -19,10 +19,19 @@ const ( // Database Object types TABLE = "table" FUNCTION = "function" + COLUMN = "column" // Source DB Types YUGABYTEDB = "yugabytedb" POSTGRESQL = "postgresql" ORACLE = "oracle" MYSQL = "mysql" + + // AssessmentIssue Categoes - used by YugabyteD payload and Migration Complexity Explainability + // TODO: soon to be renamed as SCHEMA, SCHEMA_PLPGSQL, DML_QUERY, MIGRATION_CAVEAT, "DATATYPE" + FEATURE = "feature" + DATATYPE = "datatype" + QUERY_CONSTRUCT = "query_construct" + MIGRATION_CAVEATS = "migration_caveats" + PLPGSQL_OBJECT = "plpgsql_object" ) diff --git a/yb-voyager/src/issue/issue.go b/yb-voyager/src/issue/issue.go index c30d4f837..fc4978a97 100644 --- a/yb-voyager/src/issue/issue.go +++ b/yb-voyager/src/issue/issue.go @@ -24,6 +24,7 @@ type Issue struct { Type string // (advisory_locks, index_not_supported, etc) TypeName string // for display TypeDescription string + Impact string Suggestion string GH string DocsLink string diff --git a/yb-voyager/src/utils/commonVariables.go b/yb-voyager/src/utils/commonVariables.go index b42c547d9..4463f65b6 100644 --- a/yb-voyager/src/utils/commonVariables.go +++ b/yb-voyager/src/utils/commonVariables.go @@ -103,10 +103,11 @@ type DBObject struct { type AnalyzeSchemaIssue struct { // TODO: rename IssueType to Category IssueType string `json:"IssueType"` //category: unsupported_features, unsupported_plpgsql_objects, etc - ObjectType string `json:"ObjectType"` - ObjectName string `json:"ObjectName"` Reason string `json:"Reason"` Type string `json:"-" xml:"-"` // identifier for issue type ADVISORY_LOCKS, SYSTEM_COLUMNS, etc + Impact string `json:"-" xml:"-"` + ObjectType string `json:"ObjectType"` + ObjectName string `json:"ObjectName"` SqlStatement string `json:"SqlStatement,omitempty"` FilePath string `json:"FilePath"` Suggestion string `json:"Suggestion"` From f55fc4db9e3ebb189ee01b893e57668eda7a5f35 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Mon, 6 Jan 2025 08:56:57 +0000 Subject: [PATCH 04/16] removed prints --- yb-voyager/cmd/assessMigrationCommand.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 43eb1feac..3ccc61a8c 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -31,7 +31,6 @@ import ( "syscall" "text/template" - "github.com/davecgh/go-spew/spew" "github.com/fatih/color" "github.com/samber/lo" log "github.com/sirupsen/logrus" @@ -429,10 +428,10 @@ func assessMigration() (err error) { if err != nil { log.Errorf("failed to fetch the assessment issues for migration complexity: %v", err) } - utils.PrintAndLog("number of assessment issues detected: %d\n", len(assessmentIssues)) - for i, issue := range assessmentIssues { - fmt.Printf("AssessmentIssue[%d] = %s\n\n", i, spew.Sdump(issue)) - } + log.Infof("number of assessment issues detected: %d\n", len(assessmentIssues)) + // for i, issue := range assessmentIssues { + // fmt.Printf("AssessmentIssue[%d] = %s\n\n", i, spew.Sdump(issue)) + // } utils.PrintAndLog("Migration assessment completed successfully.") completedEvent := createMigrationAssessmentCompletedEvent() From b6adfcf83f0235d22beb5af01cfddebf209cf3de Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Mon, 6 Jan 2025 10:17:57 +0000 Subject: [PATCH 05/16] minor cleanup and comments --- migtests/scripts/functions.sh | 21 +++++++++++++++++---- yb-voyager/cmd/common.go | 9 +++++---- yb-voyager/src/utils/commonVariables.go | 4 ++-- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index 18f3ff9a2..4d8e1d6f4 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -877,7 +877,13 @@ normalize_json() { # Normalize JSON with jq; use --sort-keys to avoid the need to keep the same sequence of keys in expected vs actual json jq --sort-keys 'walk( if type == "object" then - .ObjectNames? |= (if type == "string" then split(", ") | sort | join(", ") else . end) | + .ObjectNames? |= ( + if type == "string" then + split(", ") | sort | join(", ") + else + . + end + ) | .VoyagerVersion? = "IGNORED" | .TargetDBVersion? = "IGNORED" | .DbVersion? = "IGNORED" | @@ -885,10 +891,17 @@ normalize_json() { .OptimalSelectConnectionsPerNode? = "IGNORED" | .OptimalInsertConnectionsPerNode? = "IGNORED" | .RowCount? = "IGNORED" | - .SqlStatement? |= (if type == "string" then gsub("\\n"; " ") else . end) + # Replace newline characters in SqlStatement with spaces + .SqlStatement? |= ( + if type == "string" then + gsub("\\n"; " ") + else + . + end + ) elif type == "array" then - sort_by(tostring) - else + sort_by(tostring) + else . end )' "$input_file" > "$temp_file" diff --git a/yb-voyager/cmd/common.go b/yb-voyager/cmd/common.go index 47c5a7c49..26431a2ff 100644 --- a/yb-voyager/cmd/common.go +++ b/yb-voyager/cmd/common.go @@ -1195,13 +1195,14 @@ type AssessmentReport struct { MigrationCaveats []UnsupportedFeature `json:"MigrationCaveats"` } +// Fields apart from Category, CategoryDescription, TypeName and Impact will be populated only if/when available type AssessmentIssue struct { - Category string + Category string // Feature, query_constrcuts, migration_caveats CategoryDescription string - TypeName string + TypeName string // GIN Indexes, Security Invoker View TypeDescription string - Impact string - ObjectType string + Impact string // Level-1, Level-2, Level-3 (default: Level-1 ??) + ObjectType string // For datatype category, ObjectType will be datatype (for eg "geometry") ObjectName string SqlStatement string DocsLink string diff --git a/yb-voyager/src/utils/commonVariables.go b/yb-voyager/src/utils/commonVariables.go index 4463f65b6..a32a85db2 100644 --- a/yb-voyager/src/utils/commonVariables.go +++ b/yb-voyager/src/utils/commonVariables.go @@ -103,11 +103,11 @@ type DBObject struct { type AnalyzeSchemaIssue struct { // TODO: rename IssueType to Category IssueType string `json:"IssueType"` //category: unsupported_features, unsupported_plpgsql_objects, etc + ObjectType string `json:"ObjectType"` + ObjectName string `json:"ObjectName"` Reason string `json:"Reason"` Type string `json:"-" xml:"-"` // identifier for issue type ADVISORY_LOCKS, SYSTEM_COLUMNS, etc Impact string `json:"-" xml:"-"` - ObjectType string `json:"ObjectType"` - ObjectName string `json:"ObjectName"` SqlStatement string `json:"SqlStatement,omitempty"` FilePath string `json:"FilePath"` Suggestion string `json:"Suggestion"` From d4318f6a047b91c837b4e731193c11b2cbf8e6d6 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Mon, 6 Jan 2025 10:26:42 +0000 Subject: [PATCH 06/16] moved pg/omnibus schema-migration test to the correct place --- .github/workflows/misc-migtests.yml | 3 --- .github/workflows/pg-17-migtests.yml | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/misc-migtests.yml b/.github/workflows/misc-migtests.yml index 3385743b8..336202848 100644 --- a/.github/workflows/misc-migtests.yml +++ b/.github/workflows/misc-migtests.yml @@ -114,9 +114,6 @@ jobs: echo "127.0.0.1 yb-master-n1" | sudo tee -a /etc/hosts psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();" - echo "TEST: PG sample schemas (omnibus)" - migtests/scripts/run-schema-migration.sh pg/omnibus - echo "Setup the gcp credentials" migtests/scripts/gcs/create_gcs_credentials_file diff --git a/.github/workflows/pg-17-migtests.yml b/.github/workflows/pg-17-migtests.yml index fc2aba70b..27aacc0c6 100644 --- a/.github/workflows/pg-17-migtests.yml +++ b/.github/workflows/pg-17-migtests.yml @@ -117,6 +117,10 @@ jobs: if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-schema-migration.sh pg/osm + - name: "TEST: PG sample schemas (omnibus)" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-schema-migration.sh pg/omnibus + - name: "TEST: PG sample schemas (adventureworks)" if: ${{ !cancelled() && matrix.test_group == 'offline' }} run: migtests/scripts/run-schema-migration.sh pg/adventureworks From b06687968becf7adbb182b821503817a4964284f Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Mon, 6 Jan 2025 20:18:52 +0000 Subject: [PATCH 07/16] Added Type field to AssessmentIssue struct --- yb-voyager/cmd/assessMigrationCommand.go | 13 ++++++++++--- yb-voyager/cmd/assessMigrationCommandV2.go | 16 +++++++++++++--- yb-voyager/cmd/common.go | 5 +++-- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 3ccc61a8c..1d689d657 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -31,6 +31,7 @@ import ( "syscall" "text/template" + "github.com/davecgh/go-spew/spew" "github.com/fatih/color" "github.com/samber/lo" log "github.com/sirupsen/logrus" @@ -429,9 +430,15 @@ func assessMigration() (err error) { log.Errorf("failed to fetch the assessment issues for migration complexity: %v", err) } log.Infof("number of assessment issues detected: %d\n", len(assessmentIssues)) - // for i, issue := range assessmentIssues { - // fmt.Printf("AssessmentIssue[%d] = %s\n\n", i, spew.Sdump(issue)) - // } + var data string + for i, issue := range assessmentIssues { + data += fmt.Sprintf("AssessmentIssue[%d] = %s\n\n", i, spew.Sdump(issue)) + } + assessIssuesFile := filepath.Join(exportDir, "assessment_issues.txt") + err = os.WriteFile(assessIssuesFile, []byte(data), 0677) + if err != nil { + panic("failed to write to assessment_issues.txt") + } utils.PrintAndLog("Migration assessment completed successfully.") completedEvent := createMigrationAssessmentCompletedEvent() diff --git a/yb-voyager/cmd/assessMigrationCommandV2.go b/yb-voyager/cmd/assessMigrationCommandV2.go index 2a7e21e67..11bd3d07e 100644 --- a/yb-voyager/cmd/assessMigrationCommandV2.go +++ b/yb-voyager/cmd/assessMigrationCommandV2.go @@ -175,6 +175,7 @@ func fetchUnsupportedObjectTypesV2() ([]AssessmentIssue, error) { case slices.Contains(OracleUnsupportedIndexTypes, objectType): assessmentIssues = append(assessmentIssues, AssessmentIssue{ Category: constants.FEATURE, + Type: "", // TODO TypeName: UNSUPPORTED_INDEXES_FEATURE, Impact: "", // TODO ObjectType: "INDEX", @@ -183,12 +184,14 @@ func fetchUnsupportedObjectTypesV2() ([]AssessmentIssue, error) { case objectType == VIRTUAL_COLUMN: assessmentIssues = append(assessmentIssues, AssessmentIssue{ Category: constants.FEATURE, + Type: "", // TODO TypeName: VIRTUAL_COLUMNS_FEATURE, ObjectName: objectName, }) case objectType == INHERITED_TYPE: assessmentIssues = append(assessmentIssues, AssessmentIssue{ Category: constants.FEATURE, + Type: "", // TODO TypeName: INHERITED_TYPES_FEATURE, ObjectName: objectName, }) @@ -196,6 +199,7 @@ func fetchUnsupportedObjectTypesV2() ([]AssessmentIssue, error) { referenceOrTablePartitionPresent = true assessmentIssues = append(assessmentIssues, AssessmentIssue{ Category: constants.FEATURE, + Type: "", // TODO TypeName: UNSUPPORTED_PARTITIONING_METHODS_FEATURE, ObjectType: "TABLE", ObjectName: fmt.Sprintf("Table Name: %s, Partition Method: %s", objectName, objectType), @@ -235,6 +239,7 @@ func fetchUnsupportedPlPgSQLObjectsV2(schemaAnalysisReport utils.SchemaReport) [ assessmentIssues = append(assessmentIssues, AssessmentIssue{ Category: constants.PLPGSQL_OBJECT, + Type: issue.Type, TypeName: reason, Impact: issue.Impact, // TODO ObjectType: issue.ObjectType, @@ -273,7 +278,8 @@ func getAssessmentIssuesFromSchemaAnalysisReport(category string, issueType stri issues = append(issues, AssessmentIssue{ Category: category, CategoryDescription: GetCategoryDescription(category), - TypeName: analyzeIssue.Type, + Type: analyzeIssue.Type, + TypeName: analyzeIssue.Reason, TypeDescription: issueDescription, // TODO: verify Impact: analyzeIssue.Impact, ObjectType: analyzeIssue.ObjectType, @@ -347,6 +353,7 @@ func fetchUnsupportedQueryConstructsV2() ([]AssessmentIssue, error) { for _, issue := range issues { issue := AssessmentIssue{ Category: constants.QUERY_CONSTRUCT, + Type: issue.Type, TypeName: issue.TypeName, SqlStatement: issue.SqlStatement, DocsLink: issue.DocsLink, @@ -366,6 +373,7 @@ func getAssessmentIssuesForUnsupportedDatatypes(unsupportedDatatypes []utils.Tab issue := AssessmentIssue{ Category: constants.DATATYPE, CategoryDescription: GetCategoryDescription(constants.DATATYPE), + Type: colInfo.DataType, // TODO: maybe name it like "unsupported datatype - geometry" TypeName: colInfo.DataType, // TODO: maybe name it like "unsupported datatype - geometry" Impact: "", // TODO ObjectType: constants.COLUMN, @@ -394,7 +402,8 @@ func fetchMigrationCaveatAssessmentIssues(unsupportedDataTypesForLiveMigration [ issue := AssessmentIssue{ Category: constants.MIGRATION_CAVEATS, CategoryDescription: "", // TODO - TypeName: UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE, // TODO add object type in type name + Type: UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE, // TODO add object type in type name + TypeName: "", // TODO TypeDescription: UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_DESCRIPTION, ObjectType: constants.COLUMN, ObjectName: fmt.Sprintf("%s.%s.%s", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName), @@ -409,7 +418,8 @@ func fetchMigrationCaveatAssessmentIssues(unsupportedDataTypesForLiveMigration [ issue := AssessmentIssue{ Category: constants.MIGRATION_CAVEATS, CategoryDescription: "", // TODO - TypeName: UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE, // TODO add object type in type name + Type: UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE, // TODO add object type in type name + TypeName: "", // TODO TypeDescription: UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_DESCRIPTION, ObjectType: constants.COLUMN, ObjectName: fmt.Sprintf("%s.%s.%s", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName), diff --git a/yb-voyager/cmd/common.go b/yb-voyager/cmd/common.go index 26431a2ff..4bc61ac40 100644 --- a/yb-voyager/cmd/common.go +++ b/yb-voyager/cmd/common.go @@ -1197,9 +1197,10 @@ type AssessmentReport struct { // Fields apart from Category, CategoryDescription, TypeName and Impact will be populated only if/when available type AssessmentIssue struct { - Category string // Feature, query_constrcuts, migration_caveats + Category string // expected values: feature, query_constrcuts, migration_caveats, plpgsql_objects, datatytpe CategoryDescription string - TypeName string // GIN Indexes, Security Invoker View + Type string // Ex: GIN_INDEXES, SECURITY_INVOKER_VIEWS, STORED_GENERATED_COLUMNS + TypeName string // Ex: "Stored generated columns are not supported." TypeDescription string Impact string // Level-1, Level-2, Level-3 (default: Level-1 ??) ObjectType string // For datatype category, ObjectType will be datatype (for eg "geometry") From c4d920bc571177351a4f23100ffdad0d95e31253 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Mon, 6 Jan 2025 20:28:05 +0000 Subject: [PATCH 08/16] Renamed TypeName and TypeDescription field of Issues structs to Name and Description --- yb-voyager/cmd/analyzeSchema.go | 6 +- yb-voyager/cmd/assessMigrationCommand.go | 2 +- yb-voyager/cmd/assessMigrationCommandV2.go | 26 ++-- yb-voyager/cmd/common.go | 4 +- yb-voyager/src/issue/issue.go | 4 +- yb-voyager/src/query/queryissue/issues_ddl.go | 106 +++++++-------- yb-voyager/src/query/queryissue/issues_dml.go | 128 +++++++++--------- 7 files changed, 138 insertions(+), 138 deletions(-) diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 07af301cf..2cbb71f6f 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -626,10 +626,10 @@ func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fil issueType = UNSUPPORTED_PLPGSQL_OBJECTS case slices.ContainsFunc(MigrationCaveatsIssues, func(i string) bool { //Adding the MIGRATION_CAVEATS issueType of the utils.Issue for these issueInstances in MigrationCaveatsIssues - return strings.Contains(issueInstance.TypeName, i) + return strings.Contains(issueInstance.Name, i) }): issueType = MIGRATION_CAVEATS - case strings.HasPrefix(issueInstance.TypeName, UNSUPPORTED_DATATYPE): + case strings.HasPrefix(issueInstance.Name, UNSUPPORTED_DATATYPE): //Adding the UNSUPPORTED_DATATYPES issueType of the utils.Issue for these issues whose TypeName starts with "Unsupported datatype ..." issueType = UNSUPPORTED_DATATYPES } @@ -674,7 +674,7 @@ func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fil IssueType: issueType, ObjectType: issueInstance.ObjectType, ObjectName: displayObjectName, - Reason: issueInstance.TypeName, + Reason: issueInstance.Name, Type: issueInstance.Type, SqlStatement: issueInstance.SqlStatement, DocsLink: issueInstance.DocsLink, diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 1d689d657..d46e49171 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -1255,7 +1255,7 @@ func fetchUnsupportedQueryConstructs() ([]utils.UnsupportedQueryConstruct, error for _, issue := range issues { uqc := utils.UnsupportedQueryConstruct{ Query: issue.SqlStatement, - ConstructTypeName: issue.TypeName, + ConstructTypeName: issue.Name, DocsLink: issue.DocsLink, MinimumVersionsFixedIn: issue.MinimumVersionsFixedIn, } diff --git a/yb-voyager/cmd/assessMigrationCommandV2.go b/yb-voyager/cmd/assessMigrationCommandV2.go index 11bd3d07e..4c7152f21 100644 --- a/yb-voyager/cmd/assessMigrationCommandV2.go +++ b/yb-voyager/cmd/assessMigrationCommandV2.go @@ -176,7 +176,7 @@ func fetchUnsupportedObjectTypesV2() ([]AssessmentIssue, error) { assessmentIssues = append(assessmentIssues, AssessmentIssue{ Category: constants.FEATURE, Type: "", // TODO - TypeName: UNSUPPORTED_INDEXES_FEATURE, + Name: UNSUPPORTED_INDEXES_FEATURE, Impact: "", // TODO ObjectType: "INDEX", ObjectName: fmt.Sprintf("Index Name: %s, Index Type=%s", objectName, objectType), @@ -185,14 +185,14 @@ func fetchUnsupportedObjectTypesV2() ([]AssessmentIssue, error) { assessmentIssues = append(assessmentIssues, AssessmentIssue{ Category: constants.FEATURE, Type: "", // TODO - TypeName: VIRTUAL_COLUMNS_FEATURE, + Name: VIRTUAL_COLUMNS_FEATURE, ObjectName: objectName, }) case objectType == INHERITED_TYPE: assessmentIssues = append(assessmentIssues, AssessmentIssue{ Category: constants.FEATURE, Type: "", // TODO - TypeName: INHERITED_TYPES_FEATURE, + Name: INHERITED_TYPES_FEATURE, ObjectName: objectName, }) case objectType == REFERENCE_PARTITION || objectType == SYSTEM_PARTITION: @@ -200,7 +200,7 @@ func fetchUnsupportedObjectTypesV2() ([]AssessmentIssue, error) { assessmentIssues = append(assessmentIssues, AssessmentIssue{ Category: constants.FEATURE, Type: "", // TODO - TypeName: UNSUPPORTED_PARTITIONING_METHODS_FEATURE, + Name: UNSUPPORTED_PARTITIONING_METHODS_FEATURE, ObjectType: "TABLE", ObjectName: fmt.Sprintf("Table Name: %s, Partition Method: %s", objectName, objectType), }) @@ -240,7 +240,7 @@ func fetchUnsupportedPlPgSQLObjectsV2(schemaAnalysisReport utils.SchemaReport) [ assessmentIssues = append(assessmentIssues, AssessmentIssue{ Category: constants.PLPGSQL_OBJECT, Type: issue.Type, - TypeName: reason, + Name: reason, Impact: issue.Impact, // TODO ObjectType: issue.ObjectType, ObjectName: issue.ObjectName, @@ -279,8 +279,8 @@ func getAssessmentIssuesFromSchemaAnalysisReport(category string, issueType stri Category: category, CategoryDescription: GetCategoryDescription(category), Type: analyzeIssue.Type, - TypeName: analyzeIssue.Reason, - TypeDescription: issueDescription, // TODO: verify + Name: analyzeIssue.Reason, // in convertIssueInstanceToAnalyzeIssue() we assign IssueType to Reason field + Description: issueDescription, // TODO: verify Impact: analyzeIssue.Impact, ObjectType: analyzeIssue.ObjectType, ObjectName: analyzeIssue.ObjectName, @@ -354,7 +354,7 @@ func fetchUnsupportedQueryConstructsV2() ([]AssessmentIssue, error) { issue := AssessmentIssue{ Category: constants.QUERY_CONSTRUCT, Type: issue.Type, - TypeName: issue.TypeName, + Name: issue.Name, SqlStatement: issue.SqlStatement, DocsLink: issue.DocsLink, MinimumVersionFixedIn: issue.MinimumVersionsFixedIn, @@ -374,7 +374,7 @@ func getAssessmentIssuesForUnsupportedDatatypes(unsupportedDatatypes []utils.Tab Category: constants.DATATYPE, CategoryDescription: GetCategoryDescription(constants.DATATYPE), Type: colInfo.DataType, // TODO: maybe name it like "unsupported datatype - geometry" - TypeName: colInfo.DataType, // TODO: maybe name it like "unsupported datatype - geometry" + Name: colInfo.DataType, // TODO: maybe name it like "unsupported datatype - geometry" Impact: "", // TODO ObjectType: constants.COLUMN, ObjectName: qualifiedColName, @@ -403,8 +403,8 @@ func fetchMigrationCaveatAssessmentIssues(unsupportedDataTypesForLiveMigration [ Category: constants.MIGRATION_CAVEATS, CategoryDescription: "", // TODO Type: UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE, // TODO add object type in type name - TypeName: "", // TODO - TypeDescription: UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_DESCRIPTION, + Name: "", // TODO + Description: UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_DESCRIPTION, ObjectType: constants.COLUMN, ObjectName: fmt.Sprintf("%s.%s.%s", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName), DocsLink: UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, @@ -419,8 +419,8 @@ func fetchMigrationCaveatAssessmentIssues(unsupportedDataTypesForLiveMigration [ Category: constants.MIGRATION_CAVEATS, CategoryDescription: "", // TODO Type: UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE, // TODO add object type in type name - TypeName: "", // TODO - TypeDescription: UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_DESCRIPTION, + Name: "", // TODO + Description: UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_DESCRIPTION, ObjectType: constants.COLUMN, ObjectName: fmt.Sprintf("%s.%s.%s", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName), DocsLink: UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, diff --git a/yb-voyager/cmd/common.go b/yb-voyager/cmd/common.go index 4bc61ac40..064d80262 100644 --- a/yb-voyager/cmd/common.go +++ b/yb-voyager/cmd/common.go @@ -1200,8 +1200,8 @@ type AssessmentIssue struct { Category string // expected values: feature, query_constrcuts, migration_caveats, plpgsql_objects, datatytpe CategoryDescription string Type string // Ex: GIN_INDEXES, SECURITY_INVOKER_VIEWS, STORED_GENERATED_COLUMNS - TypeName string // Ex: "Stored generated columns are not supported." - TypeDescription string + Name string // Ex: "Stored generated columns are not supported." + Description string Impact string // Level-1, Level-2, Level-3 (default: Level-1 ??) ObjectType string // For datatype category, ObjectType will be datatype (for eg "geometry") ObjectName string diff --git a/yb-voyager/src/issue/issue.go b/yb-voyager/src/issue/issue.go index fc4978a97..39a6f2b50 100644 --- a/yb-voyager/src/issue/issue.go +++ b/yb-voyager/src/issue/issue.go @@ -22,8 +22,8 @@ import ( type Issue struct { Type string // (advisory_locks, index_not_supported, etc) - TypeName string // for display - TypeDescription string + Name string // for display + Description string Impact string Suggestion string GH string diff --git a/yb-voyager/src/query/queryissue/issues_ddl.go b/yb-voyager/src/query/queryissue/issues_ddl.go index 2930b5df4..faa482bbb 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl.go +++ b/yb-voyager/src/query/queryissue/issues_ddl.go @@ -26,7 +26,7 @@ import ( var generatedColumnsIssue = issue.Issue{ Type: STORED_GENERATED_COLUMNS, - TypeName: "Stored generated columns are not supported.", + Name: "Stored generated columns are not supported.", GH: "https://github.com/yugabyte/yugabyte-db/issues/10695", Suggestion: "Using Triggers to update the generated columns is one way to work around this issue, refer docs link for more details.", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#generated-always-as-stored-type-column-is-not-supported", @@ -34,13 +34,13 @@ var generatedColumnsIssue = issue.Issue{ func NewGeneratedColumnsIssue(objectType string, objectName string, sqlStatement string, generatedColumns []string) QueryIssue { issue := generatedColumnsIssue - issue.TypeName = issue.TypeName + fmt.Sprintf(" Generated Columns: (%s)", strings.Join(generatedColumns, ",")) + issue.Name = issue.Name + fmt.Sprintf(" Generated Columns: (%s)", strings.Join(generatedColumns, ",")) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var unloggedTableIssue = issue.Issue{ Type: UNLOGGED_TABLE, - TypeName: "UNLOGGED tables are not supported yet.", + Name: "UNLOGGED tables are not supported yet.", GH: "https://github.com/yugabyte/yugabyte-db/issues/1129/", Suggestion: "Remove UNLOGGED keyword to make it work", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unlogged-table-is-not-supported", @@ -56,20 +56,20 @@ func NewUnloggedTableIssue(objectType string, objectName string, sqlStatement st var unsupportedIndexMethodIssue = issue.Issue{ Type: UNSUPPORTED_INDEX_METHOD, - TypeName: "Schema contains %s index which is not supported.", + Name: "Schema contains %s index which is not supported.", GH: "https://github.com/YugaByte/yugabyte-db/issues/1337", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported", } func NewUnsupportedIndexMethodIssue(objectType string, objectName string, sqlStatement string, indexAccessMethod string) QueryIssue { issue := unsupportedIndexMethodIssue - issue.TypeName = fmt.Sprintf(unsupportedIndexMethodIssue.TypeName, strings.ToUpper(indexAccessMethod)) + issue.Name = fmt.Sprintf(unsupportedIndexMethodIssue.Name, strings.ToUpper(indexAccessMethod)) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var storageParameterIssue = issue.Issue{ Type: STORAGE_PARAMETER, - TypeName: "Storage parameters are not supported yet.", + Name: "Storage parameters are not supported yet.", GH: "https://github.com/yugabyte/yugabyte-db/issues/23467", Suggestion: "Remove the storage parameters from the DDL", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", @@ -82,7 +82,7 @@ func NewStorageParameterIssue(objectType string, objectName string, sqlStatement var setColumnAttributeIssue = issue.Issue{ Type: ALTER_TABLE_SET_COLUMN_ATTRIBUTE, - TypeName: "ALTER TABLE .. ALTER COLUMN .. SET ( attribute = value ) not supported yet", + Name: "ALTER TABLE .. ALTER COLUMN .. SET ( attribute = value ) not supported yet", GH: "https://github.com/yugabyte/yugabyte-db/issues/1124", Suggestion: "Remove it from the exported schema", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", @@ -95,7 +95,7 @@ func NewSetColumnAttributeIssue(objectType string, objectName string, sqlStateme var alterTableClusterOnIssue = issue.Issue{ Type: ALTER_TABLE_CLUSTER_ON, - TypeName: "ALTER TABLE CLUSTER not supported yet.", + Name: "ALTER TABLE CLUSTER not supported yet.", GH: "https://github.com/YugaByte/yugabyte-db/issues/1124", Suggestion: "Remove it from the exported schema.", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", @@ -108,7 +108,7 @@ func NewClusterONIssue(objectType string, objectName string, sqlStatement string var alterTableDisableRuleIssue = issue.Issue{ Type: ALTER_TABLE_DISABLE_RULE, - TypeName: "ALTER TABLE name DISABLE RULE not supported yet", + Name: "ALTER TABLE name DISABLE RULE not supported yet", GH: "https://github.com/yugabyte/yugabyte-db/issues/1124", Suggestion: "Remove this and the rule '%s' from the exported schema to be not enabled on the table.", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", @@ -123,7 +123,7 @@ func NewAlterTableDisableRuleIssue(objectType string, objectName string, sqlStat var exclusionConstraintIssue = issue.Issue{ Type: EXCLUSION_CONSTRAINTS, - TypeName: "Exclusion constraint is not supported yet", + Name: "Exclusion constraint is not supported yet", GH: "https://github.com/yugabyte/yugabyte-db/issues/3944", Suggestion: "Refer docs link for details on possible workaround", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#exclusion-constraints-is-not-supported", @@ -138,7 +138,7 @@ func NewExclusionConstraintIssue(objectType string, objectName string, sqlStatem var deferrableConstraintIssue = issue.Issue{ Type: DEFERRABLE_CONSTRAINTS, - TypeName: "DEFERRABLE constraints not supported yet", + Name: "DEFERRABLE constraints not supported yet", GH: "https://github.com/yugabyte/yugabyte-db/issues/1709", Suggestion: "Remove these constraints from the exported schema and make the neccessary changes to the application to work on target seamlessly", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#deferrable-constraint-on-constraints-other-than-foreign-keys-is-not-supported", @@ -153,7 +153,7 @@ func NewDeferrableConstraintIssue(objectType string, objectName string, sqlState var multiColumnGinIndexIssue = issue.Issue{ Type: MULTI_COLUMN_GIN_INDEX, - TypeName: "Schema contains gin index on multi column which is not supported.", + Name: "Schema contains gin index on multi column which is not supported.", GH: "https://github.com/yugabyte/yugabyte-db/issues/10652", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gin-indexes-on-multiple-columns-are-not-supported", } @@ -164,7 +164,7 @@ func NewMultiColumnGinIndexIssue(objectType string, objectName string, sqlStatem var orderedGinIndexIssue = issue.Issue{ Type: ORDERED_GIN_INDEX, - TypeName: "Schema contains gin index on column with ASC/DESC/HASH Clause which is not supported.", + Name: "Schema contains gin index on column with ASC/DESC/HASH Clause which is not supported.", GH: "https://github.com/yugabyte/yugabyte-db/issues/10653", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#issue-in-some-unsupported-cases-of-gin-indexes", } @@ -175,7 +175,7 @@ func NewOrderedGinIndexIssue(objectType string, objectName string, sqlStatement var policyRoleIssue = issue.Issue{ Type: POLICY_WITH_ROLES, - TypeName: "Policy require roles to be created.", + Name: "Policy require roles to be created.", Suggestion: "Users/Grants are not migrated during the schema migration. Create the Users manually to make the policies work", GH: "https://github.com/yugabyte/yb-voyager/issues/1655", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", @@ -183,13 +183,13 @@ var policyRoleIssue = issue.Issue{ func NewPolicyRoleIssue(objectType string, objectName string, sqlStatement string, roles []string) QueryIssue { issue := policyRoleIssue - issue.TypeName = fmt.Sprintf("%s Users - (%s)", issue.TypeName, strings.Join(roles, ",")) + issue.Name = fmt.Sprintf("%s Users - (%s)", issue.Name, strings.Join(roles, ",")) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var constraintTriggerIssue = issue.Issue{ Type: CONSTRAINT_TRIGGER, - TypeName: "CONSTRAINT TRIGGER not supported yet.", + Name: "CONSTRAINT TRIGGER not supported yet.", GH: "https://github.com/YugaByte/yugabyte-db/issues/1709", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#constraint-trigger-is-not-supported", } @@ -201,7 +201,7 @@ func NewConstraintTriggerIssue(objectType string, objectName string, sqlStatemen var referencingClauseInTriggerIssue = issue.Issue{ Type: REFERENCING_CLAUSE_IN_TRIGGER, - TypeName: "REFERENCING clause (transition tables) not supported yet.", + Name: "REFERENCING clause (transition tables) not supported yet.", GH: "https://github.com/YugaByte/yugabyte-db/issues/1668", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#referencing-clause-for-triggers", } @@ -212,7 +212,7 @@ func NewReferencingClauseTrigIssue(objectType string, objectName string, sqlStat var beforeRowTriggerOnPartitionTableIssue = issue.Issue{ Type: BEFORE_ROW_TRIGGER_ON_PARTITIONED_TABLE, - TypeName: "Partitioned tables cannot have BEFORE / FOR EACH ROW triggers.", + Name: "Partitioned tables cannot have BEFORE / FOR EACH ROW triggers.", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#before-row-triggers-on-partitioned-tables", GH: "https://github.com/yugabyte/yugabyte-db/issues/24830", Suggestion: "Create the triggers on individual partitions.", @@ -224,7 +224,7 @@ func NewBeforeRowOnPartitionTableIssue(objectType string, objectName string, sql var alterTableAddPKOnPartitionIssue = issue.Issue{ Type: ALTER_TABLE_ADD_PK_ON_PARTITIONED_TABLE, - TypeName: "Adding primary key to a partitioned table is not supported yet.", + Name: "Adding primary key to a partitioned table is not supported yet.", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#adding-primary-key-to-a-partitioned-table-results-in-an-error", GH: "https://github.com/yugabyte/yugabyte-db/issues/10074", MinimumVersionsFixedIn: map[string]*ybversion.YBVersion{ @@ -241,7 +241,7 @@ func NewAlterTableAddPKOnPartiionIssue(objectType string, objectName string, sql var expressionPartitionIssue = issue.Issue{ Type: EXPRESSION_PARTITION_WITH_PK_UK, - TypeName: "Issue with Partition using Expression on a table which cannot contain Primary Key / Unique Key on any column", + Name: "Issue with Partition using Expression on a table which cannot contain Primary Key / Unique Key on any column", Suggestion: "Remove the Constriant from the table definition", GH: "https://github.com/yugabyte/yb-voyager/issues/698", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/mysql/#tables-partitioned-with-expressions-cannot-contain-primary-unique-keys", @@ -253,7 +253,7 @@ func NewExpressionPartitionIssue(objectType string, objectName string, sqlStatem var multiColumnListPartition = issue.Issue{ Type: MULTI_COLUMN_LIST_PARTITION, - TypeName: `cannot use "list" partition strategy with more than one column`, + Name: `cannot use "list" partition strategy with more than one column`, Suggestion: "Make it a single column partition by list or choose other supported Partitioning methods", GH: "https://github.com/yugabyte/yb-voyager/issues/699", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/mysql/#multi-column-partition-by-list-is-not-supported", @@ -265,7 +265,7 @@ func NewMultiColumnListPartition(objectType string, objectName string, sqlStatem var insufficientColumnsInPKForPartition = issue.Issue{ Type: INSUFFICIENT_COLUMNS_IN_PK_FOR_PARTITION, - TypeName: "insufficient columns in the PRIMARY KEY constraint definition in CREATE TABLE", + Name: "insufficient columns in the PRIMARY KEY constraint definition in CREATE TABLE", Suggestion: "Add all Partition columns to Primary Key", GH: "https://github.com/yugabyte/yb-voyager/issues/578", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/oracle/#partition-key-column-not-part-of-primary-key-columns", @@ -273,13 +273,13 @@ var insufficientColumnsInPKForPartition = issue.Issue{ func NewInsufficientColumnInPKForPartition(objectType string, objectName string, sqlStatement string, partitionColumnsNotInPK []string) QueryIssue { issue := insufficientColumnsInPKForPartition - issue.TypeName = fmt.Sprintf("%s - (%s)", issue.TypeName, strings.Join(partitionColumnsNotInPK, ", ")) + issue.Name = fmt.Sprintf("%s - (%s)", issue.Name, strings.Join(partitionColumnsNotInPK, ", ")) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var xmlDatatypeIssue = issue.Issue{ Type: XML_DATATYPE, - TypeName: "Unsupported datatype - xml", + Name: "Unsupported datatype - xml", Suggestion: "Data ingestion is not supported for this type in YugabyteDB so handle this type in different way. Refer link for more details.", GH: "https://github.com/yugabyte/yugabyte-db/issues/1043", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported", @@ -287,13 +287,13 @@ var xmlDatatypeIssue = issue.Issue{ func NewXMLDatatypeIssue(objectType string, objectName string, sqlStatement string, colName string) QueryIssue { issue := xmlDatatypeIssue - issue.TypeName = fmt.Sprintf("%s on column - %s", issue.TypeName, colName) + issue.Name = fmt.Sprintf("%s on column - %s", issue.Name, colName) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var xidDatatypeIssue = issue.Issue{ Type: XID_DATATYPE, - TypeName: "Unsupported datatype - xid", + Name: "Unsupported datatype - xid", Suggestion: "Functions for this type e.g. txid_current are not supported in YugabyteDB yet", GH: "https://github.com/yugabyte/yugabyte-db/issues/15638", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xid-functions-is-not-supported", @@ -301,65 +301,65 @@ var xidDatatypeIssue = issue.Issue{ func NewXIDDatatypeIssue(objectType string, objectName string, sqlStatement string, colName string) QueryIssue { issue := xidDatatypeIssue - issue.TypeName = fmt.Sprintf("%s on column - %s", issue.TypeName, colName) + issue.Name = fmt.Sprintf("%s on column - %s", issue.Name, colName) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var postgisDatatypeIssue = issue.Issue{ Type: POSTGIS_DATATYPES, - TypeName: "Unsupported datatype", + Name: "Unsupported datatype", GH: "https://github.com/yugabyte/yugabyte-db/issues/11323", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb", } func NewPostGisDatatypeIssue(objectType string, objectName string, sqlStatement string, typeName string, colName string) QueryIssue { issue := postgisDatatypeIssue - issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) + issue.Name = fmt.Sprintf("%s - %s on column - %s", issue.Name, typeName, colName) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var unsupportedDatatypesIssue = issue.Issue{ Type: UNSUPPORTED_DATATYPES, - TypeName: "Unsupported datatype", + Name: "Unsupported datatype", GH: "https://github.com/yugabyte/yb-voyager/issues/1731", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb", } func NewUnsupportedDatatypesIssue(objectType string, objectName string, sqlStatement string, typeName string, colName string) QueryIssue { issue := unsupportedDatatypesIssue - issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) + issue.Name = fmt.Sprintf("%s - %s on column - %s", issue.Name, typeName, colName) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var unsupportedDatatypesForLiveMigrationIssue = issue.Issue{ Type: UNSUPPORTED_DATATYPES_LIVE_MIGRATION, - TypeName: "Unsupported datatype for Live migration", + Name: "Unsupported datatype for Live migration", GH: "https://github.com/yugabyte/yb-voyager/issues/1731", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", } func NewUnsupportedDatatypesForLMIssue(objectType string, objectName string, sqlStatement string, typeName string, colName string) QueryIssue { issue := unsupportedDatatypesForLiveMigrationIssue - issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) + issue.Name = fmt.Sprintf("%s - %s on column - %s", issue.Name, typeName, colName) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var unsupportedDatatypesForLiveMigrationWithFFOrFBIssue = issue.Issue{ Type: UNSUPPORTED_DATATYPES_LIVE_MIGRATION_WITH_FF_FB, - TypeName: "Unsupported datatype for Live migration with fall-forward/fallback", + Name: "Unsupported datatype for Live migration with fall-forward/fallback", GH: "https://github.com/yugabyte/yb-voyager/issues/1731", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", } func NewUnsupportedDatatypesForLMWithFFOrFBIssue(objectType string, objectName string, sqlStatement string, typeName string, colName string) QueryIssue { issue := unsupportedDatatypesForLiveMigrationWithFFOrFBIssue - issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) + issue.Name = fmt.Sprintf("%s - %s on column - %s", issue.Name, typeName, colName) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var primaryOrUniqueOnUnsupportedIndexTypesIssue = issue.Issue{ Type: PK_UK_ON_COMPLEX_DATATYPE, - TypeName: "Primary key and Unique constraint on column '%s' not yet supported", + Name: "Primary key and Unique constraint on column '%s' not yet supported", GH: "https://github.com/yugabyte/yugabyte-db/issues/25003", Suggestion: "Refer to the docs link for the workaround", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", //Keeping it similar for now, will see if we need to a separate issue on docs, @@ -370,13 +370,13 @@ func NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue(objectType string, objec CONSTRAINT_NAME: constraintName, } issue := primaryOrUniqueOnUnsupportedIndexTypesIssue - issue.TypeName = fmt.Sprintf(issue.TypeName, typeName) + issue.Name = fmt.Sprintf(issue.Name, typeName) return newQueryIssue(issue, objectType, objectName, sqlStatement, details) } var indexOnComplexDatatypesIssue = issue.Issue{ Type: INDEX_ON_COMPLEX_DATATYPE, - TypeName: "INDEX on column '%s' not yet supported", + Name: "INDEX on column '%s' not yet supported", GH: "https://github.com/yugabyte/yugabyte-db/issues/25003", Suggestion: "Refer to the docs link for the workaround", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", @@ -384,13 +384,13 @@ var indexOnComplexDatatypesIssue = issue.Issue{ func NewIndexOnComplexDatatypesIssue(objectType string, objectName string, sqlStatement string, typeName string) QueryIssue { issue := indexOnComplexDatatypesIssue - issue.TypeName = fmt.Sprintf(issue.TypeName, typeName) + issue.Name = fmt.Sprintf(issue.Name, typeName) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var foreignTableIssue = issue.Issue{ Type: FOREIGN_TABLE, - TypeName: "Foreign tables require manual intervention.", + Name: "Foreign tables require manual intervention.", GH: "https://github.com/yugabyte/yb-voyager/issues/1627", Suggestion: "SERVER '%s', and USER MAPPING should be created manually on the target to create and use the foreign table", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping", @@ -404,7 +404,7 @@ func NewForeignTableIssue(objectType string, objectName string, sqlStatement str var inheritanceIssue = issue.Issue{ Type: INHERITANCE, - TypeName: "TABLE INHERITANCE not supported in YugabyteDB", + Name: "TABLE INHERITANCE not supported in YugabyteDB", GH: "https://github.com/YugaByte/yugabyte-db/issues/1129", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", } @@ -414,12 +414,12 @@ func NewInheritanceIssue(objectType string, objectName string, sqlStatement stri } var percentTypeSyntax = issue.Issue{ - Type: REFERENCED_TYPE_DECLARATION, - TypeName: "Referenced type declaration of variables", - TypeDescription: "", - Suggestion: "Fix the syntax to include the actual type name instead of referencing the type of a column", - GH: "https://github.com/yugabyte/yugabyte-db/issues/23619", - DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", + Type: REFERENCED_TYPE_DECLARATION, + Name: "Referenced type declaration of variables", + Description: "", + Suggestion: "Fix the syntax to include the actual type name instead of referencing the type of a column", + GH: "https://github.com/yugabyte/yugabyte-db/issues/23619", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#type-syntax-is-not-supported", } func NewPercentTypeSyntaxIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -428,7 +428,7 @@ func NewPercentTypeSyntaxIssue(objectType string, objectName string, sqlStatemen var loDatatypeIssue = issue.Issue{ Type: LARGE_OBJECT_DATATYPE, - TypeName: "Unsupported datatype - lo", + Name: "Unsupported datatype - lo", Suggestion: "Large objects are not yet supported in YugabyteDB, no workaround available currently", GH: "https://github.com/yugabyte/yugabyte-db/issues/25318", DocsLink: "", // TODO @@ -436,13 +436,13 @@ var loDatatypeIssue = issue.Issue{ func NewLODatatypeIssue(objectType string, objectName string, SqlStatement string, colName string) QueryIssue { issue := loDatatypeIssue - issue.TypeName = fmt.Sprintf("%s on column - %s", issue.TypeName, colName) + issue.Name = fmt.Sprintf("%s on column - %s", issue.Name, colName) return newQueryIssue(issue, objectType, objectName, SqlStatement, map[string]interface{}{}) } var multiRangeDatatypeIssue = issue.Issue{ Type: MULTI_RANGE_DATATYPE, - TypeName: "Unsupported datatype", + Name: "Unsupported datatype", Suggestion: "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", GH: "", //TODO DocsLink: "", //TODO @@ -450,13 +450,13 @@ var multiRangeDatatypeIssue = issue.Issue{ func NewMultiRangeDatatypeIssue(objectType string, objectName string, sqlStatement string, typeName string, colName string) QueryIssue { issue := multiRangeDatatypeIssue - issue.TypeName = fmt.Sprintf("%s - %s on column - %s", issue.TypeName, typeName, colName) + issue.Name = fmt.Sprintf("%s - %s on column - %s", issue.Name, typeName, colName) return newQueryIssue(issue, objectType, objectName, sqlStatement, map[string]interface{}{}) } var securityInvokerViewIssue = issue.Issue{ Type: SECURITY_INVOKER_VIEWS, - TypeName: "Security Invoker Views not supported yet", + Name: "Security Invoker Views not supported yet", Suggestion: "Security Invoker Views are not yet supported in YugabyteDB, no workaround available currently", GH: "", // TODO DocsLink: "", // TODO @@ -468,7 +468,7 @@ func NewSecurityInvokerViewIssue(objectType string, objectName string, SqlStatem var foreignKeyReferencesPartitionedTableIssue = issue.Issue{ Type: FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE, - TypeName: FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME, + Name: FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME, Suggestion: "No workaround available ", GH: "", // TODO DocsLink: "", // TODO diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index e9b8f2592..679ce64e7 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -23,12 +23,12 @@ import ( ) var advisoryLocksIssue = issue.Issue{ - Type: ADVISORY_LOCKS, - TypeName: "Advisory Locks", - TypeDescription: "", - Suggestion: "", - GH: "", - DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", + Type: ADVISORY_LOCKS, + Name: "Advisory Locks", + Description: "", + Suggestion: "", + GH: "", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#advisory-locks-is-not-yet-implemented", } func NewAdvisoryLocksIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -36,12 +36,12 @@ func NewAdvisoryLocksIssue(objectType string, objectName string, sqlStatement st } var systemColumnsIssue = issue.Issue{ - Type: SYSTEM_COLUMNS, - TypeName: "System Columns", - TypeDescription: "", - Suggestion: "", - GH: "", - DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported", + Type: SYSTEM_COLUMNS, + Name: "System Columns", + Description: "", + Suggestion: "", + GH: "", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#system-columns-is-not-yet-supported", } func NewSystemColumnsIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -49,12 +49,12 @@ func NewSystemColumnsIssue(objectType string, objectName string, sqlStatement st } var xmlFunctionsIssue = issue.Issue{ - Type: XML_FUNCTIONS, - TypeName: "XML Functions", - TypeDescription: "", - Suggestion: "", - GH: "", - DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", + Type: XML_FUNCTIONS, + Name: "XML Functions", + Description: "", + Suggestion: "", + GH: "", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xml-functions-is-not-yet-supported", } func NewXmlFunctionsIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -62,12 +62,12 @@ func NewXmlFunctionsIssue(objectType string, objectName string, sqlStatement str } var regexFunctionsIssue = issue.Issue{ - Type: REGEX_FUNCTIONS, - TypeName: "Regex Functions", - TypeDescription: "", - Suggestion: "", - GH: "", - DocsLink: "", + Type: REGEX_FUNCTIONS, + Name: "Regex Functions", + Description: "", + Suggestion: "", + GH: "", + DocsLink: "", } func NewRegexFunctionsIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -76,8 +76,8 @@ func NewRegexFunctionsIssue(objectType string, objectName string, sqlStatement s var aggregateFunctionIssue = issue.Issue{ Type: AGGREGATE_FUNCTION, - TypeName: AGGREGATION_FUNCTIONS_NAME, - TypeDescription: "any_value, range_agg and range_intersect_agg functions not supported yet in YugabyteDB", + Name: AGGREGATION_FUNCTIONS_NAME, + Description: "any_value, range_agg and range_intersect_agg functions not supported yet in YugabyteDB", Suggestion: "", GH: "", DocsLink: "", @@ -92,12 +92,12 @@ func NewAggregationFunctionIssue(objectType string, objectName string, sqlStatem } var jsonConstructorFunctionsIssue = issue.Issue{ - Type: JSON_CONSTRUCTOR_FUNCTION, - TypeName: JSON_CONSTRUCTOR_FUNCTION_NAME, - TypeDescription: "Postgresql 17 features not supported yet in YugabyteDB", - Suggestion: "", - GH: "", - DocsLink: "", + Type: JSON_CONSTRUCTOR_FUNCTION, + Name: JSON_CONSTRUCTOR_FUNCTION_NAME, + Description: "Postgresql 17 features not supported yet in YugabyteDB", + Suggestion: "", + GH: "", + DocsLink: "", } func NewJsonConstructorFunctionIssue(objectType string, objectName string, sqlStatement string, funcNames []string) QueryIssue { @@ -109,12 +109,12 @@ func NewJsonConstructorFunctionIssue(objectType string, objectName string, sqlSt } var jsonQueryFunctionIssue = issue.Issue{ - Type: JSON_QUERY_FUNCTION, - TypeName: JSON_QUERY_FUNCTIONS_NAME, - TypeDescription: "Postgresql 17 features not supported yet in YugabyteDB", - Suggestion: "", - GH: "", - DocsLink: "", + Type: JSON_QUERY_FUNCTION, + Name: JSON_QUERY_FUNCTIONS_NAME, + Description: "Postgresql 17 features not supported yet in YugabyteDB", + Suggestion: "", + GH: "", + DocsLink: "", } func NewJsonQueryFunctionIssue(objectType string, objectName string, sqlStatement string, funcNames []string) QueryIssue { @@ -126,12 +126,12 @@ func NewJsonQueryFunctionIssue(objectType string, objectName string, sqlStatemen } var loFunctionsIssue = issue.Issue{ - Type: LARGE_OBJECT_FUNCTIONS, - TypeName: LARGE_OBJECT_FUNCTIONS_NAME, - TypeDescription: "Large Objects functions are not supported in YugabyteDB", - Suggestion: "Large objects functions are not yet supported in YugabyteDB, no workaround available right now", - GH: "https://github.com/yugabyte/yugabyte-db/issues/25318", - DocsLink: "", //TODO + Type: LARGE_OBJECT_FUNCTIONS, + Name: LARGE_OBJECT_FUNCTIONS_NAME, + Description: "Large Objects functions are not supported in YugabyteDB", + Suggestion: "Large objects functions are not yet supported in YugabyteDB, no workaround available right now", + GH: "https://github.com/yugabyte/yugabyte-db/issues/25318", + DocsLink: "", //TODO } func NewLOFuntionsIssue(objectType string, objectName string, sqlStatement string, funcNames []string) QueryIssue { @@ -157,8 +157,8 @@ func NewJsonbSubscriptingIssue(objectType string, objectName string, sqlStatemen var jsonPredicateIssue = issue.Issue{ Type: JSON_TYPE_PREDICATE, - TypeName: JSON_TYPE_PREDICATE_NAME, - TypeDescription: "IS JSON predicate expressions not supported yet in YugabyteDB", + Name: JSON_TYPE_PREDICATE_NAME, + Description: "IS JSON predicate expressions not supported yet in YugabyteDB", Suggestion: "", GH: "", DocsLink: "", //TODO @@ -169,12 +169,12 @@ func NewJsonPredicateIssue(objectType string, objectName string, sqlStatement st } var copyFromWhereIssue = issue.Issue{ - Type: COPY_FROM_WHERE, - TypeName: "COPY FROM ... WHERE", - TypeDescription: "", - Suggestion: "", - GH: "", - DocsLink: "", + Type: COPY_FROM_WHERE, + Name: "COPY FROM ... WHERE", + Description: "", + Suggestion: "", + GH: "", + DocsLink: "", } func NewCopyFromWhereIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -182,12 +182,12 @@ func NewCopyFromWhereIssue(objectType string, objectName string, sqlStatement st } var copyOnErrorIssue = issue.Issue{ - Type: COPY_ON_ERROR, - TypeName: "COPY ... ON_ERROR", - TypeDescription: "", - Suggestion: "", - GH: "", - DocsLink: "", + Type: COPY_ON_ERROR, + Name: "COPY ... ON_ERROR", + Description: "", + Suggestion: "", + GH: "", + DocsLink: "", } func NewCopyOnErrorIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -195,12 +195,12 @@ func NewCopyOnErrorIssue(objectType string, objectName string, sqlStatement stri } var fetchWithTiesIssue = issue.Issue{ - Type: FETCH_WITH_TIES, - TypeName: "FETCH .. WITH TIES", - TypeDescription: "FETCH .. WITH TIES is not supported in YugabyteDB", - Suggestion: "No workaround available right now", - GH: "", - DocsLink: "", //TODO + Type: FETCH_WITH_TIES, + Name: "FETCH .. WITH TIES", + Description: "FETCH .. WITH TIES is not supported in YugabyteDB", + Suggestion: "No workaround available right now", + GH: "", + DocsLink: "", //TODO } func NewFetchWithTiesIssue(objectType string, objectName string, sqlStatement string) QueryIssue { From ae2802729ebc010165b01f527e632efa9f937e65 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Mon, 6 Jan 2025 21:29:03 +0000 Subject: [PATCH 09/16] Defining impact for issues_ddl and issues_dml and propagating that to AssessmentIssue --- yb-voyager/cmd/analyzeSchema.go | 1 + yb-voyager/src/constants/constants.go | 5 +++ yb-voyager/src/issue/issue.go | 41 +++++++++++++++++++ yb-voyager/src/query/queryissue/issues_ddl.go | 34 +++++++++++++++ yb-voyager/src/query/queryissue/issues_dml.go | 7 ++++ yb-voyager/src/utils/commonVariables.go | 2 +- 6 files changed, 89 insertions(+), 1 deletion(-) diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 2cbb71f6f..75435a3c4 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -676,6 +676,7 @@ func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fil ObjectName: displayObjectName, Reason: issueInstance.Name, Type: issueInstance.Type, + Impact: issueInstance.GetImpact(), SqlStatement: issueInstance.SqlStatement, DocsLink: issueInstance.DocsLink, FilePath: fileName, diff --git a/yb-voyager/src/constants/constants.go b/yb-voyager/src/constants/constants.go index 59710a9e4..809d1a301 100644 --- a/yb-voyager/src/constants/constants.go +++ b/yb-voyager/src/constants/constants.go @@ -34,4 +34,9 @@ const ( QUERY_CONSTRUCT = "query_construct" MIGRATION_CAVEATS = "migration_caveats" PLPGSQL_OBJECT = "plpgsql_object" + + // constants for the Impact Buckets + IMPACT_LEVEL_1 = "LEVEL_1" // Represents minimal impact like only the schema ddl + IMPACT_LEVEL_2 = "LEVEL_2" // Represents moderate impact like dml queries which might impact a lot of implementation/assumption in app layer + IMPACT_LEVEL_3 = "LEVEL_3" // Represent significant impact like TABLE INHERITANCE, which doesn't have any simple workaround but can impact multiple objects/apps ) diff --git a/yb-voyager/src/issue/issue.go b/yb-voyager/src/issue/issue.go index 39a6f2b50..f1c9358c2 100644 --- a/yb-voyager/src/issue/issue.go +++ b/yb-voyager/src/issue/issue.go @@ -17,6 +17,8 @@ limitations under the License. package issue import ( + "github.com/samber/lo" + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" ) @@ -41,3 +43,42 @@ func (i Issue) IsFixedIn(v *ybversion.YBVersion) (bool, error) { } return v.GreaterThanOrEqual(minVersionFixedInSeries), nil } + +func (i Issue) GetImpact() string { + // Default value as IMPACT_LEVEL_1 if not defined in issue + return lo.Ternary(i.Impact != "", i.Impact, constants.IMPACT_LEVEL_1) +} + +/* + Dynamic Impact Determination (TODO) + - We can define the impact calculator function based on issue type wherever/whenever needed + - Map will have functions only for issue type with dynamic impact determination + + For example: + + type ImpactCalcFunc func(issue QueryIssue, stats *PgStats) string + + var impactCalculators = map[string]ImpactCalcFunc{ + INHERITED_TABLE: inheritedTableImpactCalc, + // etc... + } + + // Example dynamic function + func inheritedTableImpactCalc(i QueryIssue, stats *PgStats) string { + usage := stats.GetUsage(i.ObjectName) // e.g. how many reads/writes + if usage.WritesPerDay > 1000 { + return "LEVEL_2" + } + return "LEVEL_3" + } + + // Update existing GetImpact() method + func (i Issue) GetImpact(stats *PgStats) string { + if calc, ok := impactCalculators[i.Type]; ok { + return calc(i, stats) + } + + return lo.Ternary(i.Impact != "", i.Impact, constants.IMPACT_LEVEL_1) + } + +*/ diff --git a/yb-voyager/src/query/queryissue/issues_ddl.go b/yb-voyager/src/query/queryissue/issues_ddl.go index faa482bbb..1187ec810 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl.go +++ b/yb-voyager/src/query/queryissue/issues_ddl.go @@ -20,6 +20,7 @@ import ( "fmt" "strings" + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" ) @@ -27,6 +28,7 @@ import ( var generatedColumnsIssue = issue.Issue{ Type: STORED_GENERATED_COLUMNS, Name: "Stored generated columns are not supported.", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yugabyte-db/issues/10695", Suggestion: "Using Triggers to update the generated columns is one way to work around this issue, refer docs link for more details.", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#generated-always-as-stored-type-column-is-not-supported", @@ -41,6 +43,7 @@ func NewGeneratedColumnsIssue(objectType string, objectName string, sqlStatement var unloggedTableIssue = issue.Issue{ Type: UNLOGGED_TABLE, Name: "UNLOGGED tables are not supported yet.", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yugabyte-db/issues/1129/", Suggestion: "Remove UNLOGGED keyword to make it work", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unlogged-table-is-not-supported", @@ -57,6 +60,7 @@ func NewUnloggedTableIssue(objectType string, objectName string, sqlStatement st var unsupportedIndexMethodIssue = issue.Issue{ Type: UNSUPPORTED_INDEX_METHOD, Name: "Schema contains %s index which is not supported.", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/YugaByte/yugabyte-db/issues/1337", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gist-brin-and-spgist-index-types-are-not-supported", } @@ -70,6 +74,7 @@ func NewUnsupportedIndexMethodIssue(objectType string, objectName string, sqlSta var storageParameterIssue = issue.Issue{ Type: STORAGE_PARAMETER, Name: "Storage parameters are not supported yet.", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yugabyte-db/issues/23467", Suggestion: "Remove the storage parameters from the DDL", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#storage-parameters-on-indexes-or-constraints-in-the-source-postgresql", @@ -83,6 +88,7 @@ func NewStorageParameterIssue(objectType string, objectName string, sqlStatement var setColumnAttributeIssue = issue.Issue{ Type: ALTER_TABLE_SET_COLUMN_ATTRIBUTE, Name: "ALTER TABLE .. ALTER COLUMN .. SET ( attribute = value ) not supported yet", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yugabyte-db/issues/1124", Suggestion: "Remove it from the exported schema", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", @@ -96,6 +102,7 @@ func NewSetColumnAttributeIssue(objectType string, objectName string, sqlStateme var alterTableClusterOnIssue = issue.Issue{ Type: ALTER_TABLE_CLUSTER_ON, Name: "ALTER TABLE CLUSTER not supported yet.", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/YugaByte/yugabyte-db/issues/1124", Suggestion: "Remove it from the exported schema.", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", @@ -109,6 +116,7 @@ func NewClusterONIssue(objectType string, objectName string, sqlStatement string var alterTableDisableRuleIssue = issue.Issue{ Type: ALTER_TABLE_DISABLE_RULE, Name: "ALTER TABLE name DISABLE RULE not supported yet", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yugabyte-db/issues/1124", Suggestion: "Remove this and the rule '%s' from the exported schema to be not enabled on the table.", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-alter-table-ddl-variants-in-source-schema", @@ -124,6 +132,7 @@ func NewAlterTableDisableRuleIssue(objectType string, objectName string, sqlStat var exclusionConstraintIssue = issue.Issue{ Type: EXCLUSION_CONSTRAINTS, Name: "Exclusion constraint is not supported yet", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yugabyte-db/issues/3944", Suggestion: "Refer docs link for details on possible workaround", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#exclusion-constraints-is-not-supported", @@ -139,6 +148,7 @@ func NewExclusionConstraintIssue(objectType string, objectName string, sqlStatem var deferrableConstraintIssue = issue.Issue{ Type: DEFERRABLE_CONSTRAINTS, Name: "DEFERRABLE constraints not supported yet", + Impact: constants.IMPACT_LEVEL_3, GH: "https://github.com/yugabyte/yugabyte-db/issues/1709", Suggestion: "Remove these constraints from the exported schema and make the neccessary changes to the application to work on target seamlessly", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#deferrable-constraint-on-constraints-other-than-foreign-keys-is-not-supported", @@ -154,6 +164,7 @@ func NewDeferrableConstraintIssue(objectType string, objectName string, sqlState var multiColumnGinIndexIssue = issue.Issue{ Type: MULTI_COLUMN_GIN_INDEX, Name: "Schema contains gin index on multi column which is not supported.", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yugabyte-db/issues/10652", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#gin-indexes-on-multiple-columns-are-not-supported", } @@ -165,6 +176,7 @@ func NewMultiColumnGinIndexIssue(objectType string, objectName string, sqlStatem var orderedGinIndexIssue = issue.Issue{ Type: ORDERED_GIN_INDEX, Name: "Schema contains gin index on column with ASC/DESC/HASH Clause which is not supported.", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yugabyte-db/issues/10653", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#issue-in-some-unsupported-cases-of-gin-indexes", } @@ -176,6 +188,7 @@ func NewOrderedGinIndexIssue(objectType string, objectName string, sqlStatement var policyRoleIssue = issue.Issue{ Type: POLICY_WITH_ROLES, Name: "Policy require roles to be created.", + Impact: constants.IMPACT_LEVEL_1, Suggestion: "Users/Grants are not migrated during the schema migration. Create the Users manually to make the policies work", GH: "https://github.com/yugabyte/yb-voyager/issues/1655", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#policies-on-users-in-source-require-manual-user-creation", @@ -190,6 +203,7 @@ func NewPolicyRoleIssue(objectType string, objectName string, sqlStatement strin var constraintTriggerIssue = issue.Issue{ Type: CONSTRAINT_TRIGGER, Name: "CONSTRAINT TRIGGER not supported yet.", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/YugaByte/yugabyte-db/issues/1709", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#constraint-trigger-is-not-supported", } @@ -202,6 +216,7 @@ func NewConstraintTriggerIssue(objectType string, objectName string, sqlStatemen var referencingClauseInTriggerIssue = issue.Issue{ Type: REFERENCING_CLAUSE_IN_TRIGGER, Name: "REFERENCING clause (transition tables) not supported yet.", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/YugaByte/yugabyte-db/issues/1668", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#referencing-clause-for-triggers", } @@ -213,6 +228,7 @@ func NewReferencingClauseTrigIssue(objectType string, objectName string, sqlStat var beforeRowTriggerOnPartitionTableIssue = issue.Issue{ Type: BEFORE_ROW_TRIGGER_ON_PARTITIONED_TABLE, Name: "Partitioned tables cannot have BEFORE / FOR EACH ROW triggers.", + Impact: constants.IMPACT_LEVEL_1, DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#before-row-triggers-on-partitioned-tables", GH: "https://github.com/yugabyte/yugabyte-db/issues/24830", Suggestion: "Create the triggers on individual partitions.", @@ -225,6 +241,7 @@ func NewBeforeRowOnPartitionTableIssue(objectType string, objectName string, sql var alterTableAddPKOnPartitionIssue = issue.Issue{ Type: ALTER_TABLE_ADD_PK_ON_PARTITIONED_TABLE, Name: "Adding primary key to a partitioned table is not supported yet.", + Impact: constants.IMPACT_LEVEL_1, DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#adding-primary-key-to-a-partitioned-table-results-in-an-error", GH: "https://github.com/yugabyte/yugabyte-db/issues/10074", MinimumVersionsFixedIn: map[string]*ybversion.YBVersion{ @@ -242,6 +259,7 @@ func NewAlterTableAddPKOnPartiionIssue(objectType string, objectName string, sql var expressionPartitionIssue = issue.Issue{ Type: EXPRESSION_PARTITION_WITH_PK_UK, Name: "Issue with Partition using Expression on a table which cannot contain Primary Key / Unique Key on any column", + Impact: constants.IMPACT_LEVEL_1, Suggestion: "Remove the Constriant from the table definition", GH: "https://github.com/yugabyte/yb-voyager/issues/698", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/mysql/#tables-partitioned-with-expressions-cannot-contain-primary-unique-keys", @@ -254,6 +272,7 @@ func NewExpressionPartitionIssue(objectType string, objectName string, sqlStatem var multiColumnListPartition = issue.Issue{ Type: MULTI_COLUMN_LIST_PARTITION, Name: `cannot use "list" partition strategy with more than one column`, + Impact: constants.IMPACT_LEVEL_1, Suggestion: "Make it a single column partition by list or choose other supported Partitioning methods", GH: "https://github.com/yugabyte/yb-voyager/issues/699", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/mysql/#multi-column-partition-by-list-is-not-supported", @@ -266,6 +285,7 @@ func NewMultiColumnListPartition(objectType string, objectName string, sqlStatem var insufficientColumnsInPKForPartition = issue.Issue{ Type: INSUFFICIENT_COLUMNS_IN_PK_FOR_PARTITION, Name: "insufficient columns in the PRIMARY KEY constraint definition in CREATE TABLE", + Impact: constants.IMPACT_LEVEL_1, Suggestion: "Add all Partition columns to Primary Key", GH: "https://github.com/yugabyte/yb-voyager/issues/578", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/oracle/#partition-key-column-not-part-of-primary-key-columns", @@ -280,6 +300,7 @@ func NewInsufficientColumnInPKForPartition(objectType string, objectName string, var xmlDatatypeIssue = issue.Issue{ Type: XML_DATATYPE, Name: "Unsupported datatype - xml", + Impact: constants.IMPACT_LEVEL_3, Suggestion: "Data ingestion is not supported for this type in YugabyteDB so handle this type in different way. Refer link for more details.", GH: "https://github.com/yugabyte/yugabyte-db/issues/1043", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#data-ingestion-on-xml-data-type-is-not-supported", @@ -294,6 +315,7 @@ func NewXMLDatatypeIssue(objectType string, objectName string, sqlStatement stri var xidDatatypeIssue = issue.Issue{ Type: XID_DATATYPE, Name: "Unsupported datatype - xid", + Impact: constants.IMPACT_LEVEL_3, Suggestion: "Functions for this type e.g. txid_current are not supported in YugabyteDB yet", GH: "https://github.com/yugabyte/yugabyte-db/issues/15638", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#xid-functions-is-not-supported", @@ -308,6 +330,7 @@ func NewXIDDatatypeIssue(objectType string, objectName string, sqlStatement stri var postgisDatatypeIssue = issue.Issue{ Type: POSTGIS_DATATYPES, Name: "Unsupported datatype", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yugabyte-db/issues/11323", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb", } @@ -321,6 +344,7 @@ func NewPostGisDatatypeIssue(objectType string, objectName string, sqlStatement var unsupportedDatatypesIssue = issue.Issue{ Type: UNSUPPORTED_DATATYPES, Name: "Unsupported datatype", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yb-voyager/issues/1731", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb", } @@ -334,6 +358,7 @@ func NewUnsupportedDatatypesIssue(objectType string, objectName string, sqlState var unsupportedDatatypesForLiveMigrationIssue = issue.Issue{ Type: UNSUPPORTED_DATATYPES_LIVE_MIGRATION, Name: "Unsupported datatype for Live migration", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yb-voyager/issues/1731", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", } @@ -347,6 +372,7 @@ func NewUnsupportedDatatypesForLMIssue(objectType string, objectName string, sql var unsupportedDatatypesForLiveMigrationWithFFOrFBIssue = issue.Issue{ Type: UNSUPPORTED_DATATYPES_LIVE_MIGRATION_WITH_FF_FB, Name: "Unsupported datatype for Live migration with fall-forward/fallback", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yb-voyager/issues/1731", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", } @@ -360,6 +386,7 @@ func NewUnsupportedDatatypesForLMWithFFOrFBIssue(objectType string, objectName s var primaryOrUniqueOnUnsupportedIndexTypesIssue = issue.Issue{ Type: PK_UK_ON_COMPLEX_DATATYPE, Name: "Primary key and Unique constraint on column '%s' not yet supported", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yugabyte-db/issues/25003", Suggestion: "Refer to the docs link for the workaround", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", //Keeping it similar for now, will see if we need to a separate issue on docs, @@ -377,6 +404,7 @@ func NewPrimaryOrUniqueConsOnUnsupportedIndexTypesIssue(objectType string, objec var indexOnComplexDatatypesIssue = issue.Issue{ Type: INDEX_ON_COMPLEX_DATATYPE, Name: "INDEX on column '%s' not yet supported", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yugabyte-db/issues/25003", Suggestion: "Refer to the docs link for the workaround", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#indexes-on-some-complex-data-types-are-not-supported", @@ -391,6 +419,7 @@ func NewIndexOnComplexDatatypesIssue(objectType string, objectName string, sqlSt var foreignTableIssue = issue.Issue{ Type: FOREIGN_TABLE, Name: "Foreign tables require manual intervention.", + Impact: constants.IMPACT_LEVEL_1, GH: "https://github.com/yugabyte/yb-voyager/issues/1627", Suggestion: "SERVER '%s', and USER MAPPING should be created manually on the target to create and use the foreign table", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#foreign-table-in-the-source-database-requires-server-and-user-mapping", @@ -405,6 +434,7 @@ func NewForeignTableIssue(objectType string, objectName string, sqlStatement str var inheritanceIssue = issue.Issue{ Type: INHERITANCE, Name: "TABLE INHERITANCE not supported in YugabyteDB", + Impact: constants.IMPACT_LEVEL_3, GH: "https://github.com/YugaByte/yugabyte-db/issues/1129", DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#table-inheritance-is-not-supported", } @@ -416,6 +446,7 @@ func NewInheritanceIssue(objectType string, objectName string, sqlStatement stri var percentTypeSyntax = issue.Issue{ Type: REFERENCED_TYPE_DECLARATION, Name: "Referenced type declaration of variables", + Impact: constants.IMPACT_LEVEL_1, Description: "", Suggestion: "Fix the syntax to include the actual type name instead of referencing the type of a column", GH: "https://github.com/yugabyte/yugabyte-db/issues/23619", @@ -429,6 +460,7 @@ func NewPercentTypeSyntaxIssue(objectType string, objectName string, sqlStatemen var loDatatypeIssue = issue.Issue{ Type: LARGE_OBJECT_DATATYPE, Name: "Unsupported datatype - lo", + Impact: constants.IMPACT_LEVEL_1, Suggestion: "Large objects are not yet supported in YugabyteDB, no workaround available currently", GH: "https://github.com/yugabyte/yugabyte-db/issues/25318", DocsLink: "", // TODO @@ -443,6 +475,7 @@ func NewLODatatypeIssue(objectType string, objectName string, SqlStatement strin var multiRangeDatatypeIssue = issue.Issue{ Type: MULTI_RANGE_DATATYPE, Name: "Unsupported datatype", + Impact: constants.IMPACT_LEVEL_1, Suggestion: "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", GH: "", //TODO DocsLink: "", //TODO @@ -457,6 +490,7 @@ func NewMultiRangeDatatypeIssue(objectType string, objectName string, sqlStateme var securityInvokerViewIssue = issue.Issue{ Type: SECURITY_INVOKER_VIEWS, Name: "Security Invoker Views not supported yet", + Impact: constants.IMPACT_LEVEL_1, Suggestion: "Security Invoker Views are not yet supported in YugabyteDB, no workaround available currently", GH: "", // TODO DocsLink: "", // TODO diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index 679ce64e7..c1ac35092 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -19,12 +19,14 @@ package queryissue import ( "sort" + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" "github.com/yugabyte/yb-voyager/yb-voyager/src/issue" ) var advisoryLocksIssue = issue.Issue{ Type: ADVISORY_LOCKS, Name: "Advisory Locks", + Impact: constants.IMPACT_LEVEL_2, Description: "", Suggestion: "", GH: "", @@ -38,6 +40,7 @@ func NewAdvisoryLocksIssue(objectType string, objectName string, sqlStatement st var systemColumnsIssue = issue.Issue{ Type: SYSTEM_COLUMNS, Name: "System Columns", + Impact: constants.IMPACT_LEVEL_2, Description: "", Suggestion: "", GH: "", @@ -51,6 +54,7 @@ func NewSystemColumnsIssue(objectType string, objectName string, sqlStatement st var xmlFunctionsIssue = issue.Issue{ Type: XML_FUNCTIONS, Name: "XML Functions", + Impact: constants.IMPACT_LEVEL_2, Description: "", Suggestion: "", GH: "", @@ -64,6 +68,7 @@ func NewXmlFunctionsIssue(objectType string, objectName string, sqlStatement str var regexFunctionsIssue = issue.Issue{ Type: REGEX_FUNCTIONS, Name: "Regex Functions", + Impact: constants.IMPACT_LEVEL_2, Description: "", Suggestion: "", GH: "", @@ -171,6 +176,7 @@ func NewJsonPredicateIssue(objectType string, objectName string, sqlStatement st var copyFromWhereIssue = issue.Issue{ Type: COPY_FROM_WHERE, Name: "COPY FROM ... WHERE", + Impact: constants.IMPACT_LEVEL_2, Description: "", Suggestion: "", GH: "", @@ -184,6 +190,7 @@ func NewCopyFromWhereIssue(objectType string, objectName string, sqlStatement st var copyOnErrorIssue = issue.Issue{ Type: COPY_ON_ERROR, Name: "COPY ... ON_ERROR", + Impact: constants.IMPACT_LEVEL_2, Description: "", Suggestion: "", GH: "", diff --git a/yb-voyager/src/utils/commonVariables.go b/yb-voyager/src/utils/commonVariables.go index a32a85db2..16a8725ad 100644 --- a/yb-voyager/src/utils/commonVariables.go +++ b/yb-voyager/src/utils/commonVariables.go @@ -107,7 +107,7 @@ type AnalyzeSchemaIssue struct { ObjectName string `json:"ObjectName"` Reason string `json:"Reason"` Type string `json:"-" xml:"-"` // identifier for issue type ADVISORY_LOCKS, SYSTEM_COLUMNS, etc - Impact string `json:"-" xml:"-"` + Impact string `json:"-" xml:"-"` // temporary field; since currently we generate assessment issue from analyze issue SqlStatement string `json:"SqlStatement,omitempty"` FilePath string `json:"FilePath"` Suggestion string `json:"Suggestion"` From 869fe094345e38173f69ee8d3baf3507ad1543a1 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Mon, 6 Jan 2025 21:46:08 +0000 Subject: [PATCH 10/16] Added new arg - impact to reportCase() for assigning impact to the issues detected using regexp in analyze schema --- yb-voyager/cmd/analyzeSchema.go | 112 ++++++++++++++++---------------- 1 file changed, 57 insertions(+), 55 deletions(-) diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 75435a3c4..79efec3b8 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -34,6 +34,7 @@ import ( "golang.org/x/exp/slices" "github.com/yugabyte/yb-voyager/yb-voyager/src/callhome" + "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" "github.com/yugabyte/yb-voyager/yb-voyager/src/cp" "github.com/yugabyte/yb-voyager/yb-voyager/src/metadb" "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryissue" @@ -220,7 +221,7 @@ const ( ) // Reports one case in JSON -func reportCase(filePath string, reason string, ghIssue string, suggestion string, objType string, objName string, sqlStmt string, issueType string, docsLink string) { +func reportCase(filePath string, reason string, ghIssue string, suggestion string, objType string, objName string, sqlStmt string, issueType string, docsLink string, impact string) { var issue utils.AnalyzeSchemaIssue issue.FilePath = filePath issue.Reason = reason @@ -230,6 +231,7 @@ func reportCase(filePath string, reason string, ghIssue string, suggestion strin issue.ObjectName = objName issue.SqlStatement = sqlStmt issue.IssueType = issueType + issue.Impact = lo.Ternary(impact != "", impact, constants.IMPACT_LEVEL_1) if sourceDBType == POSTGRESQL { issue.DocsLink = docsLink } @@ -239,13 +241,13 @@ func reportCase(filePath string, reason string, ghIssue string, suggestion strin func reportBasedOnComment(comment int, fpath string, issue string, suggestion string, objName string, objType string, line string) { if comment == 1 { - reportCase(fpath, "Unsupported, please edit to match PostgreSQL syntax", "https://github.com/yugabyte/yb-voyager/issues/1625", suggestion, objType, objName, line, UNSUPPORTED_FEATURES, "") + reportCase(fpath, "Unsupported, please edit to match PostgreSQL syntax", "https://github.com/yugabyte/yb-voyager/issues/1625", suggestion, objType, objName, line, UNSUPPORTED_FEATURES, "", "") summaryMap[objType].invalidCount[objName] = true } else if comment == 2 { - // reportCase(fpath, "PACKAGE in oracle are exported as Schema, please review and edit to match PostgreSQL syntax if required, Package is "+objName, issue, suggestion, objType) + // reportCase(fpath, "PACKAGE in oracle are exported as Schema, please review and edit to match PostgreSQL syntax if required, Package is "+objName, issue, suggestion, objType, "") summaryMap["PACKAGE"].objSet = append(summaryMap["PACKAGE"].objSet, objName) } else if comment == 3 { - reportCase(fpath, "SQLs in file might be unsupported please review and edit to match PostgreSQL syntax if required. ", "https://github.com/yugabyte/yb-voyager/issues/1625", suggestion, objType, objName, line, UNSUPPORTED_FEATURES, "") + reportCase(fpath, "SQLs in file might be unsupported please review and edit to match PostgreSQL syntax if required. ", "https://github.com/yugabyte/yb-voyager/issues/1625", suggestion, objType, objName, line, UNSUPPORTED_FEATURES, "", "") } else if comment == 4 { summaryMap[objType].details["Inherited Types are present which are not supported in PostgreSQL syntax, so exported as Inherited Tables"] = true } @@ -321,7 +323,7 @@ func checkStmtsUsingParser(sqlInfoArr []sqlInfo, fpath string, objType string) { if !summaryMap[objType].invalidCount[sqlStmtInfo.objName] { reason := fmt.Sprintf("%s - '%s'", UNSUPPORTED_PG_SYNTAX, err.Error()) reportCase(fpath, reason, "https://github.com/yugabyte/yb-voyager/issues/1625", - "Fix the schema as per PG syntax", objType, sqlStmtInfo.objName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "Fix the schema as per PG syntax", objType, sqlStmtInfo.objName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } continue } @@ -348,7 +350,7 @@ func checkViews(sqlInfoArr []sqlInfo, fpath string) { if view := viewWithCheckRegex.FindStringSubmatch(sqlInfo.stmt); view != nil { summaryMap["VIEW"].invalidCount[sqlInfo.objName] = true reportCase(fpath, VIEW_CHECK_OPTION_ISSUE, "https://github.com/yugabyte/yugabyte-db/issues/22716", - "Use Trigger with INSTEAD OF clause on INSERT/UPDATE on view to get this functionality", "VIEW", view[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, VIEW_CHECK_OPTION_DOC_LINK) + "Use Trigger with INSTEAD OF clause on INSERT/UPDATE on view to get this functionality", "VIEW", view[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, VIEW_CHECK_OPTION_DOC_LINK, "") } } } @@ -372,47 +374,47 @@ func checkSql(sqlInfoArr []sqlInfo, fpath string) { if rangeRegex.MatchString(sqlInfo.stmt) { reportCase(fpath, "RANGE with offset PRECEDING/FOLLOWING is not supported for column type numeric and offset type double precision", - "https://github.com/yugabyte/yugabyte-db/issues/10692", "", "TABLE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/yugabyte/yugabyte-db/issues/10692", "", "TABLE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true } else if stmt := fetchRegex.FindStringSubmatch(sqlInfo.stmt); stmt != nil { location := strings.ToUpper(stmt[1]) if slices.Contains(notSupportedFetchLocation, location) { summaryMap["PROCEDURE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "This FETCH clause might not be supported yet", "https://github.com/YugaByte/yugabyte-db/issues/6514", - "Please verify the DDL on your YugabyteDB version before proceeding", "CURSOR", sqlInfo.objName, sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "Please verify the DDL on your YugabyteDB version before proceeding", "CURSOR", sqlInfo.objName, sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } } else if stmt := alterAggRegex.FindStringSubmatch(sqlInfo.stmt); stmt != nil { summaryMap["AGGREGATE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER AGGREGATE not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/2717", "", "AGGREGATE", stmt[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/2717", "", "AGGREGATE", stmt[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if dropCollRegex.MatchString(sqlInfo.stmt) { summaryMap["COLLATION"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP COLLATION", sqlInfo.formattedStmt), "COLLATION", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP COLLATION", sqlInfo.formattedStmt), "COLLATION", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if dropIdxRegex.MatchString(sqlInfo.stmt) { summaryMap["INDEX"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP INDEX", sqlInfo.formattedStmt), "INDEX", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP INDEX", sqlInfo.formattedStmt), "INDEX", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if dropViewRegex.MatchString(sqlInfo.stmt) { summaryMap["VIEW"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP VIEW", sqlInfo.formattedStmt), "VIEW", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP VIEW", sqlInfo.formattedStmt), "VIEW", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if dropSeqRegex.MatchString(sqlInfo.stmt) { summaryMap["SEQUENCE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP SEQUENCE", sqlInfo.formattedStmt), "SEQUENCE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP SEQUENCE", sqlInfo.formattedStmt), "SEQUENCE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if dropForeignRegex.MatchString(sqlInfo.stmt) { summaryMap["FOREIGN TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP FOREIGN TABLE", sqlInfo.formattedStmt), "FOREIGN TABLE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP FOREIGN TABLE", sqlInfo.formattedStmt), "FOREIGN TABLE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if idx := dropIdxConcurRegex.FindStringSubmatch(sqlInfo.stmt); idx != nil { summaryMap["INDEX"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP INDEX CONCURRENTLY not supported yet", - "https://github.com/yugabyte/yugabyte-db/issues/22717", "", "INDEX", idx[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/yugabyte/yugabyte-db/issues/22717", "", "INDEX", idx[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if currentOfRegex.MatchString(sqlInfo.stmt) { - reportCase(fpath, "WHERE CURRENT OF not supported yet", "https://github.com/YugaByte/yugabyte-db/issues/737", "", "CURSOR", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + reportCase(fpath, "WHERE CURRENT OF not supported yet", "https://github.com/YugaByte/yugabyte-db/issues/737", "", "CURSOR", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if bulkCollectRegex.MatchString(sqlInfo.stmt) { - reportCase(fpath, "BULK COLLECT keyword of oracle is not converted into PostgreSQL compatible syntax", "https://github.com/yugabyte/yb-voyager/issues/1539", "", "", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + reportCase(fpath, "BULK COLLECT keyword of oracle is not converted into PostgreSQL compatible syntax", "https://github.com/yugabyte/yb-voyager/issues/1539", "", "", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } } } @@ -424,106 +426,106 @@ func checkDDL(sqlInfoArr []sqlInfo, fpath string, objType string) { if am := amRegex.FindStringSubmatch(sqlInfo.stmt); am != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "CREATE ACCESS METHOD is not supported.", - "https://github.com/yugabyte/yugabyte-db/issues/10693", "", "ACCESS METHOD", am[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/yugabyte/yugabyte-db/issues/10693", "", "ACCESS METHOD", am[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if tbl := idxConcRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "REINDEX is not supported.", - "https://github.com/yugabyte/yugabyte-db/issues/10267", "", "TABLE", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/yugabyte/yugabyte-db/issues/10267", "", "TABLE", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if tbl := likeAllRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "LIKE ALL is not supported yet.", - "https://github.com/yugabyte/yugabyte-db/issues/10697", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/yugabyte/yugabyte-db/issues/10697", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if tbl := likeRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "LIKE clause not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1129", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1129", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if tbl := withOidsRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "OIDs are not supported for user tables.", - "https://github.com/yugabyte/yugabyte-db/issues/10273", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/yugabyte/yugabyte-db/issues/10273", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if tbl := alterOfRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE OF not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if tbl := alterSchemaRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE SET SCHEMA not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/3947", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/3947", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if createSchemaRegex.MatchString(sqlInfo.stmt) { summaryMap["SCHEMA"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "CREATE SCHEMA with elements not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/10865", "", "SCHEMA", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/10865", "", "SCHEMA", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if tbl := alterNotOfRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE NOT OF not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if tbl := alterColumnStatsRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE ALTER column SET STATISTICS not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if tbl := alterColumnStorageRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE ALTER column SET STORAGE not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if tbl := alterColumnResetAttributesRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE ALTER column RESET (attribute) not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if tbl := alterConstrRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE ALTER CONSTRAINT not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if tbl := setOidsRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE SET WITH OIDS not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[4], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[4], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if tbl := withoutClusterRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE SET WITHOUT CLUSTER not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if tbl := alterSetRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE SET not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if tbl := alterIdxRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER INDEX SET not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "INDEX", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "INDEX", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if tbl := alterResetRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE RESET not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if tbl := alterOptionsRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if typ := dropAttrRegex.FindStringSubmatch(sqlInfo.stmt); typ != nil { summaryMap["TYPE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TYPE DROP ATTRIBUTE not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1893", "", "TYPE", typ[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1893", "", "TYPE", typ[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if typ := alterTypeRegex.FindStringSubmatch(sqlInfo.stmt); typ != nil { summaryMap["TYPE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TYPE not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1893", "", "TYPE", typ[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1893", "", "TYPE", typ[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if tbl := alterInhRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE INHERIT not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if tbl := valConstrRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE VALIDATE CONSTRAINT not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if spc := alterTblSpcRegex.FindStringSubmatch(sqlInfo.stmt); spc != nil { summaryMap["TABLESPACE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLESPACE not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1153", "", "TABLESPACE", spc[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1153", "", "TABLESPACE", spc[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if spc := alterViewRegex.FindStringSubmatch(sqlInfo.stmt); spc != nil { summaryMap["VIEW"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER VIEW not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1131", "", "VIEW", spc[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/YugaByte/yugabyte-db/issues/1131", "", "VIEW", spc[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if tbl := cLangRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { reportCase(fpath, "LANGUAGE C not supported yet.", - "https://github.com/yugabyte/yb-voyager/issues/1540", "", "FUNCTION", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/yugabyte/yb-voyager/issues/1540", "", "FUNCTION", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") summaryMap["FUNCTION"].invalidCount[sqlInfo.objName] = true } else if strings.Contains(strings.ToLower(sqlInfo.stmt), "drop temporary table") { filePath := strings.Split(fpath, "/") @@ -531,22 +533,22 @@ func checkDDL(sqlInfoArr []sqlInfo, fpath string, objType string) { objType := strings.ToUpper(strings.Split(fileName, ".")[0]) summaryMap[objType].invalidCount[sqlInfo.objName] = true reportCase(fpath, `temporary table is not a supported clause for drop`, - "https://github.com/yugabyte/yb-voyager/issues/705", `remove "temporary" and change it to "drop table"`, objType, sqlInfo.objName, sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, DROP_TEMP_TABLE_DOC_LINK) + "https://github.com/yugabyte/yb-voyager/issues/705", `remove "temporary" and change it to "drop table"`, objType, sqlInfo.objName, sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, DROP_TEMP_TABLE_DOC_LINK, "") } else if regMatch := anydataRegex.FindStringSubmatch(sqlInfo.stmt); regMatch != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true - reportCase(fpath, "AnyData datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with AnyData datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + reportCase(fpath, "AnyData datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with AnyData datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if regMatch := anydatasetRegex.FindStringSubmatch(sqlInfo.stmt); regMatch != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true - reportCase(fpath, "AnyDataSet datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with AnyDataSet datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + reportCase(fpath, "AnyDataSet datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with AnyDataSet datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if regMatch := anyTypeRegex.FindStringSubmatch(sqlInfo.stmt); regMatch != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true - reportCase(fpath, "AnyType datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with AnyType datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + reportCase(fpath, "AnyType datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with AnyType datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if regMatch := uriTypeRegex.FindStringSubmatch(sqlInfo.stmt); regMatch != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true - reportCase(fpath, "URIType datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with URIType datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + reportCase(fpath, "URIType datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with URIType datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if regMatch := jsonFuncRegex.FindStringSubmatch(sqlInfo.stmt); regMatch != nil { summaryMap[objType].invalidCount[sqlInfo.objName] = true - reportCase(fpath, "JSON_ARRAYAGG() function is not available in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1542", `Rename the function to YugabyteDB's equivalent JSON_AGG()`, objType, regMatch[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + reportCase(fpath, "JSON_ARRAYAGG() function is not available in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1542", `Rename the function to YugabyteDB's equivalent JSON_AGG()`, objType, regMatch[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } } @@ -559,11 +561,11 @@ func checkForeign(sqlInfoArr []sqlInfo, fpath string) { if tbl := primRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "Primary key constraints are not supported on foreign tables.", - "https://github.com/yugabyte/yugabyte-db/issues/10698", "", "TABLE", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/yugabyte/yugabyte-db/issues/10698", "", "TABLE", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } else if tbl := foreignKeyRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "Foreign key constraints are not supported on foreign tables.", - "https://github.com/yugabyte/yugabyte-db/issues/10699", "", "TABLE", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/yugabyte/yugabyte-db/issues/10699", "", "TABLE", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") } } } @@ -573,7 +575,7 @@ func checkRemaining(sqlInfoArr []sqlInfo, fpath string) { for _, sqlInfo := range sqlInfoArr { if trig := compoundTrigRegex.FindStringSubmatch(sqlInfo.stmt); trig != nil { reportCase(fpath, COMPOUND_TRIGGER_ISSUE_REASON, - "https://github.com/yugabyte/yb-voyager/issues/1543", "", "TRIGGER", trig[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "") + "https://github.com/yugabyte/yb-voyager/issues/1543", "", "TRIGGER", trig[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") summaryMap["TRIGGER"].invalidCount[sqlInfo.objName] = true } } @@ -691,7 +693,7 @@ func checkExtensions(sqlInfoArr []sqlInfo, fpath string) { if sqlInfo.objName != "" && !slices.Contains(supportedExtensionsOnYB, sqlInfo.objName) { summaryMap["EXTENSION"].invalidCount[sqlInfo.objName] = true reportCase(fpath, UNSUPPORTED_EXTENSION_ISSUE+" Refer to the docs link for the more information on supported extensions.", "https://github.com/yugabyte/yb-voyager/issues/1538", "", "EXTENSION", - sqlInfo.objName, sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, EXTENSION_DOC_LINK) + sqlInfo.objName, sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, EXTENSION_DOC_LINK, constants.IMPACT_LEVEL_3) } if strings.ToLower(sqlInfo.objName) == "hll" { summaryMap["EXTENSION"].details[`'hll' extension is supported in YugabyteDB v2.18 onwards. Please verify this extension as per the target YugabyteDB version.`] = true @@ -1111,12 +1113,12 @@ func checkConversions(sqlInfoArr []sqlInfo, filePath string) { convName = fmt.Sprintf("%s.%s", convName, nameList[1].GetString_().Sval) } reportCase(filePath, CONVERSION_ISSUE_REASON, "https://github.com/yugabyte/yugabyte-db/issues/10866", - "Remove it from the exported schema", "CONVERSION", convName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, CREATE_CONVERSION_DOC_LINK) + "Remove it from the exported schema", "CONVERSION", convName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, CREATE_CONVERSION_DOC_LINK, constants.IMPACT_LEVEL_3) } else { //pg_query doesn't seem to have a Node type of AlterConversionStmt so using regex for now if stmt := alterConvRegex.FindStringSubmatch(sqlStmtInfo.stmt); stmt != nil { reportCase(filePath, "ALTER CONVERSION is not supported yet", "https://github.com/YugaByte/yugabyte-db/issues/10866", - "Remove it from the exported schema", "CONVERSION", stmt[1], sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, CREATE_CONVERSION_DOC_LINK) + "Remove it from the exported schema", "CONVERSION", stmt[1], sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, CREATE_CONVERSION_DOC_LINK, "") } } From 54f1d59d844db87431d7d407c1ea5221638d338d Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Mon, 6 Jan 2025 21:56:11 +0000 Subject: [PATCH 11/16] Added placeholder for impact of the new categories being considered now --- yb-voyager/cmd/assessMigrationCommandV2.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/yb-voyager/cmd/assessMigrationCommandV2.go b/yb-voyager/cmd/assessMigrationCommandV2.go index 4c7152f21..dfae96b51 100644 --- a/yb-voyager/cmd/assessMigrationCommandV2.go +++ b/yb-voyager/cmd/assessMigrationCommandV2.go @@ -186,6 +186,7 @@ func fetchUnsupportedObjectTypesV2() ([]AssessmentIssue, error) { Category: constants.FEATURE, Type: "", // TODO Name: VIRTUAL_COLUMNS_FEATURE, + Impact: "", // TODO ObjectName: objectName, }) case objectType == INHERITED_TYPE: @@ -193,6 +194,7 @@ func fetchUnsupportedObjectTypesV2() ([]AssessmentIssue, error) { Category: constants.FEATURE, Type: "", // TODO Name: INHERITED_TYPES_FEATURE, + Impact: "", // TODO ObjectName: objectName, }) case objectType == REFERENCE_PARTITION || objectType == SYSTEM_PARTITION: @@ -201,6 +203,7 @@ func fetchUnsupportedObjectTypesV2() ([]AssessmentIssue, error) { Category: constants.FEATURE, Type: "", // TODO Name: UNSUPPORTED_PARTITIONING_METHODS_FEATURE, + Impact: "", // TODO ObjectType: "TABLE", ObjectName: fmt.Sprintf("Table Name: %s, Partition Method: %s", objectName, objectType), }) @@ -241,7 +244,7 @@ func fetchUnsupportedPlPgSQLObjectsV2(schemaAnalysisReport utils.SchemaReport) [ Category: constants.PLPGSQL_OBJECT, Type: issue.Type, Name: reason, - Impact: issue.Impact, // TODO + Impact: issue.Impact, // TODO: verify(expected already there since underlying issues are assigned) ObjectType: issue.ObjectType, ObjectName: issue.ObjectName, SqlStatement: issue.SqlStatement, @@ -355,6 +358,7 @@ func fetchUnsupportedQueryConstructsV2() ([]AssessmentIssue, error) { Category: constants.QUERY_CONSTRUCT, Type: issue.Type, Name: issue.Name, + Impact: issue.Impact, SqlStatement: issue.SqlStatement, DocsLink: issue.DocsLink, MinimumVersionFixedIn: issue.MinimumVersionsFixedIn, @@ -404,6 +408,7 @@ func fetchMigrationCaveatAssessmentIssues(unsupportedDataTypesForLiveMigration [ CategoryDescription: "", // TODO Type: UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE, // TODO add object type in type name Name: "", // TODO + Impact: "", // TODO Description: UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_DESCRIPTION, ObjectType: constants.COLUMN, ObjectName: fmt.Sprintf("%s.%s.%s", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName), @@ -420,6 +425,7 @@ func fetchMigrationCaveatAssessmentIssues(unsupportedDataTypesForLiveMigration [ CategoryDescription: "", // TODO Type: UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE, // TODO add object type in type name Name: "", // TODO + Impact: "", // TODO Description: UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_DESCRIPTION, ObjectType: constants.COLUMN, ObjectName: fmt.Sprintf("%s.%s.%s", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName), From 5cd1f84633cfb754282ca45c2beb6520072a5014 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Tue, 7 Jan 2025 08:32:07 +0000 Subject: [PATCH 12/16] Assigning impact to remaining cases/issues --- yb-voyager/cmd/assessMigrationCommandV2.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/yb-voyager/cmd/assessMigrationCommandV2.go b/yb-voyager/cmd/assessMigrationCommandV2.go index dfae96b51..15c0c6bc3 100644 --- a/yb-voyager/cmd/assessMigrationCommandV2.go +++ b/yb-voyager/cmd/assessMigrationCommandV2.go @@ -171,13 +171,13 @@ func fetchUnsupportedObjectTypesV2() ([]AssessmentIssue, error) { return nil, fmt.Errorf("error scanning rows:%w", err) } + // For these oracle issues defining Impact not required, since oracle migration complexity comes from ora2pg switch { case slices.Contains(OracleUnsupportedIndexTypes, objectType): assessmentIssues = append(assessmentIssues, AssessmentIssue{ Category: constants.FEATURE, Type: "", // TODO Name: UNSUPPORTED_INDEXES_FEATURE, - Impact: "", // TODO ObjectType: "INDEX", ObjectName: fmt.Sprintf("Index Name: %s, Index Type=%s", objectName, objectType), }) @@ -186,7 +186,6 @@ func fetchUnsupportedObjectTypesV2() ([]AssessmentIssue, error) { Category: constants.FEATURE, Type: "", // TODO Name: VIRTUAL_COLUMNS_FEATURE, - Impact: "", // TODO ObjectName: objectName, }) case objectType == INHERITED_TYPE: @@ -194,7 +193,6 @@ func fetchUnsupportedObjectTypesV2() ([]AssessmentIssue, error) { Category: constants.FEATURE, Type: "", // TODO Name: INHERITED_TYPES_FEATURE, - Impact: "", // TODO ObjectName: objectName, }) case objectType == REFERENCE_PARTITION || objectType == SYSTEM_PARTITION: @@ -203,7 +201,6 @@ func fetchUnsupportedObjectTypesV2() ([]AssessmentIssue, error) { Category: constants.FEATURE, Type: "", // TODO Name: UNSUPPORTED_PARTITIONING_METHODS_FEATURE, - Impact: "", // TODO ObjectType: "TABLE", ObjectName: fmt.Sprintf("Table Name: %s, Partition Method: %s", objectName, objectType), }) @@ -379,7 +376,7 @@ func getAssessmentIssuesForUnsupportedDatatypes(unsupportedDatatypes []utils.Tab CategoryDescription: GetCategoryDescription(constants.DATATYPE), Type: colInfo.DataType, // TODO: maybe name it like "unsupported datatype - geometry" Name: colInfo.DataType, // TODO: maybe name it like "unsupported datatype - geometry" - Impact: "", // TODO + Impact: constants.IMPACT_LEVEL_3, ObjectType: constants.COLUMN, ObjectName: qualifiedColName, DocsLink: "", // TODO @@ -408,7 +405,7 @@ func fetchMigrationCaveatAssessmentIssues(unsupportedDataTypesForLiveMigration [ CategoryDescription: "", // TODO Type: UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE, // TODO add object type in type name Name: "", // TODO - Impact: "", // TODO + Impact: constants.IMPACT_LEVEL_1, // Caveat - we don't know the migration is offline/online; Description: UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_DESCRIPTION, ObjectType: constants.COLUMN, ObjectName: fmt.Sprintf("%s.%s.%s", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName), @@ -425,7 +422,7 @@ func fetchMigrationCaveatAssessmentIssues(unsupportedDataTypesForLiveMigration [ CategoryDescription: "", // TODO Type: UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE, // TODO add object type in type name Name: "", // TODO - Impact: "", // TODO + Impact: constants.IMPACT_LEVEL_1, Description: UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_DESCRIPTION, ObjectType: constants.COLUMN, ObjectName: fmt.Sprintf("%s.%s.%s", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName), From cf943b2c2d390e9bd1de964156c0bba9019bafbb Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Tue, 7 Jan 2025 10:27:30 +0000 Subject: [PATCH 13/16] Moved code for building AssessmentIssue to the main codepath - having V2 file and function will increase the scope to maintain the code --- yb-voyager/cmd/assessMigrationCommand.go | 225 ++++++--- yb-voyager/cmd/assessMigrationCommandV2.go | 437 ------------------ yb-voyager/cmd/common.go | 4 + yb-voyager/src/query/queryissue/issues_dml.go | 28 +- 4 files changed, 186 insertions(+), 508 deletions(-) delete mode 100644 yb-voyager/cmd/assessMigrationCommandV2.go diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index d46e49171..c5463d134 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -31,7 +31,6 @@ import ( "syscall" "text/template" - "github.com/davecgh/go-spew/spew" "github.com/fatih/color" "github.com/samber/lo" log "github.com/sirupsen/logrus" @@ -425,20 +424,16 @@ func assessMigration() (err error) { return fmt.Errorf("failed to generate assessment report: %w", err) } - assessmentIssues, err := fetchAllAssessmentIssues() - if err != nil { - log.Errorf("failed to fetch the assessment issues for migration complexity: %v", err) - } - log.Infof("number of assessment issues detected: %d\n", len(assessmentIssues)) - var data string - for i, issue := range assessmentIssues { - data += fmt.Sprintf("AssessmentIssue[%d] = %s\n\n", i, spew.Sdump(issue)) - } - assessIssuesFile := filepath.Join(exportDir, "assessment_issues.txt") - err = os.WriteFile(assessIssuesFile, []byte(data), 0677) - if err != nil { - panic("failed to write to assessment_issues.txt") - } + log.Infof("number of assessment issues detected: %d\n", len(assessmentReport.Issues)) + // var data string + // for i, issue := range assessmentReport.Issues { + // data += fmt.Sprintf("AssessmentIssue[%d] = %s\n\n", i, spew.Sdump(issue)) + // } + // assessIssuesFile := filepath.Join(exportDir, "assessment_issues.txt") + // err = os.WriteFile(assessIssuesFile, []byte(data), 0677) + // if err != nil { + // panic("failed to write to assessment_issues.txt") + // } utils.PrintAndLog("Migration assessment completed successfully.") completedEvent := createMigrationAssessmentCompletedEvent() @@ -907,6 +902,8 @@ func generateAssessmentReport() (err error) { assessmentReport.UnsupportedDataTypes = unsupportedDataTypes assessmentReport.UnsupportedDataTypesDesc = DATATYPE_CATEGORY_DESCRIPTION + assessmentReport.AppendIssues(getAssessmentIssuesForUnsupportedDatatypes(unsupportedDataTypes)...) + addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration, unsupportedDataTypesForLiveMigrationWithFForFB) assessmentReport.Sizing = migassessment.SizingReport @@ -988,7 +985,7 @@ func areMinVersionsFixedInEqual(m1 map[string]*ybversion.YBVersion, m2 map[strin return true } -func getUnsupportedFeaturesFromSchemaAnalysisReport(featureName string, issueReason string, issueType string, schemaAnalysisReport utils.SchemaReport, displayDDLInHTML bool, description string) UnsupportedFeature { +func getUnsupportedFeaturesFromSchemaAnalysisReport(category string, featureName string, issueReason string, issueType string, schemaAnalysisReport utils.SchemaReport, displayDDLInHTML bool, description string) UnsupportedFeature { log.Info("filtering issues for feature: ", featureName) objects := make([]ObjectInfo, 0) link := "" // for oracle we shouldn't display any line for links @@ -1016,12 +1013,31 @@ func getUnsupportedFeaturesFromSchemaAnalysisReport(featureName string, issueRea } link = analyzeIssue.DocsLink objects = append(objects, objectInfo) + + assessmentReport.AppendIssues(convertAnalyzeSchemaIssueToAssessmentIssue(analyzeIssue, category, description, minVersionsFixedIn)) } } return UnsupportedFeature{featureName, objects, displayDDLInHTML, link, description, minVersionsFixedIn} } +// Q: do we no need of displayDDLInHTML in this approach? DDL can always be there for issues in the table if present. +func convertAnalyzeSchemaIssueToAssessmentIssue(analyzeSchemaIssue utils.AnalyzeSchemaIssue, category string, issueDescription string, minVersionsFixedIn map[string]*ybversion.YBVersion) AssessmentIssue { + return AssessmentIssue{ + Category: category, + CategoryDescription: GetCategoryDescription(category), + Type: analyzeSchemaIssue.Type, + Name: analyzeSchemaIssue.Reason, // in convertIssueInstanceToAnalyzeIssue() we assign IssueType to Reason field + Description: issueDescription, // TODO: verify + Impact: analyzeSchemaIssue.Impact, + ObjectType: analyzeSchemaIssue.ObjectType, + ObjectName: analyzeSchemaIssue.ObjectName, + SqlStatement: analyzeSchemaIssue.SqlStatement, + DocsLink: analyzeSchemaIssue.DocsLink, + MinimumVersionFixedIn: minVersionsFixedIn, + } +} + func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.SchemaReport) ([]UnsupportedFeature, error) { log.Infof("fetching unsupported features for PG...") unsupportedFeatures := make([]UnsupportedFeature, 0) @@ -1030,39 +1046,39 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem displayIndexMethod := strings.ToUpper(indexMethod) featureName := fmt.Sprintf("%s indexes", displayIndexMethod) reason := fmt.Sprintf(INDEX_METHOD_ISSUE_REASON, displayIndexMethod) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(featureName, reason, "", schemaAnalysisReport, false, "")) - } - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CONSTRAINT_TRIGGERS_FEATURE, "", queryissue.CONSTRAINT_TRIGGER, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(INHERITED_TABLES_FEATURE, "", queryissue.INHERITANCE, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(GENERATED_COLUMNS_FEATURE, "", queryissue.STORED_GENERATED_COLUMNS, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CONVERSIONS_OBJECTS_FEATURE, CONVERSION_ISSUE_REASON, "", schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(MULTI_COLUMN_GIN_INDEX_FEATURE, "", queryissue.MULTI_COLUMN_GIN_INDEX, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(ALTER_SETTING_ATTRIBUTE_FEATURE, "", queryissue.ALTER_TABLE_SET_COLUMN_ATTRIBUTE, schemaAnalysisReport, true, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(DISABLING_TABLE_RULE_FEATURE, "", queryissue.ALTER_TABLE_DISABLE_RULE, schemaAnalysisReport, true, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CLUSTER_ON_FEATURE, "", queryissue.ALTER_TABLE_CLUSTER_ON, schemaAnalysisReport, true, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(STORAGE_PARAMETERS_FEATURE, "", queryissue.STORAGE_PARAMETER, schemaAnalysisReport, true, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(EXTENSION_FEATURE, UNSUPPORTED_EXTENSION_ISSUE, "", schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(EXCLUSION_CONSTRAINT_FEATURE, "", queryissue.EXCLUSION_CONSTRAINTS, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(DEFERRABLE_CONSTRAINT_FEATURE, "", queryissue.DEFERRABLE_CONSTRAINTS, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(VIEW_CHECK_FEATURE, VIEW_CHECK_OPTION_ISSUE, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, featureName, reason, "", schemaAnalysisReport, false, "")) + } + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, CONSTRAINT_TRIGGERS_FEATURE, "", queryissue.CONSTRAINT_TRIGGER, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, INHERITED_TABLES_FEATURE, "", queryissue.INHERITANCE, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, GENERATED_COLUMNS_FEATURE, "", queryissue.STORED_GENERATED_COLUMNS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, CONVERSIONS_OBJECTS_FEATURE, CONVERSION_ISSUE_REASON, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, MULTI_COLUMN_GIN_INDEX_FEATURE, "", queryissue.MULTI_COLUMN_GIN_INDEX, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, ALTER_SETTING_ATTRIBUTE_FEATURE, "", queryissue.ALTER_TABLE_SET_COLUMN_ATTRIBUTE, schemaAnalysisReport, true, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, DISABLING_TABLE_RULE_FEATURE, "", queryissue.ALTER_TABLE_DISABLE_RULE, schemaAnalysisReport, true, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, CLUSTER_ON_FEATURE, "", queryissue.ALTER_TABLE_CLUSTER_ON, schemaAnalysisReport, true, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, STORAGE_PARAMETERS_FEATURE, "", queryissue.STORAGE_PARAMETER, schemaAnalysisReport, true, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, EXTENSION_FEATURE, UNSUPPORTED_EXTENSION_ISSUE, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, EXCLUSION_CONSTRAINT_FEATURE, "", queryissue.EXCLUSION_CONSTRAINTS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, DEFERRABLE_CONSTRAINT_FEATURE, "", queryissue.DEFERRABLE_CONSTRAINTS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, VIEW_CHECK_FEATURE, VIEW_CHECK_OPTION_ISSUE, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisReport, queryissue.UnsupportedIndexDatatypes)) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(PK_UK_CONSTRAINT_ON_COMPLEX_DATATYPES_FEATURE, "", queryissue.PK_UK_ON_COMPLEX_DATATYPE, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(UNLOGGED_TABLE_FEATURE, "", queryissue.UNLOGGED_TABLE, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REFERENCING_TRIGGER_FEATURE, "", queryissue.REFERENCING_CLAUSE_IN_TRIGGER, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE_FEATURE, "", queryissue.BEFORE_ROW_TRIGGER_ON_PARTITIONED_TABLE, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.ADVISORY_LOCKS_NAME, "", queryissue.ADVISORY_LOCKS, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.XML_FUNCTIONS_NAME, "", queryissue.XML_FUNCTIONS, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SYSTEM_COLUMNS_NAME, "", queryissue.SYSTEM_COLUMNS, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.LARGE_OBJECT_FUNCTIONS_NAME, "", queryissue.LARGE_OBJECT_FUNCTIONS, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REGEX_FUNCTIONS_FEATURE, "", queryissue.REGEX_FUNCTIONS, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(FETCH_WITH_TIES_FEATURE, "", queryissue.FETCH_WITH_TIES, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_QUERY_FUNCTIONS_NAME, "", queryissue.JSON_QUERY_FUNCTION, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_CONSTRUCTOR_FUNCTION_NAME, "", queryissue.JSON_CONSTRUCTOR_FUNCTION, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.AGGREGATION_FUNCTIONS_NAME, "", queryissue.AGGREGATE_FUNCTION, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SECURITY_INVOKER_VIEWS_NAME, "", queryissue.SECURITY_INVOKER_VIEWS, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSONB_SUBSCRIPTING_NAME, "", queryissue.JSONB_SUBSCRIPTING, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME, "", queryissue.FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_TYPE_PREDICATE_NAME, "", queryissue.JSON_TYPE_PREDICATE, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, PK_UK_CONSTRAINT_ON_COMPLEX_DATATYPES_FEATURE, "", queryissue.PK_UK_ON_COMPLEX_DATATYPE, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, UNLOGGED_TABLE_FEATURE, "", queryissue.UNLOGGED_TABLE, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, REFERENCING_TRIGGER_FEATURE, "", queryissue.REFERENCING_CLAUSE_IN_TRIGGER, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE_FEATURE, "", queryissue.BEFORE_ROW_TRIGGER_ON_PARTITIONED_TABLE, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, queryissue.ADVISORY_LOCKS_NAME, "", queryissue.ADVISORY_LOCKS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, queryissue.XML_FUNCTIONS_NAME, "", queryissue.XML_FUNCTIONS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, queryissue.SYSTEM_COLUMNS_NAME, "", queryissue.SYSTEM_COLUMNS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, queryissue.LARGE_OBJECT_FUNCTIONS_NAME, "", queryissue.LARGE_OBJECT_FUNCTIONS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, REGEX_FUNCTIONS_FEATURE, "", queryissue.REGEX_FUNCTIONS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, FETCH_WITH_TIES_FEATURE, "", queryissue.FETCH_WITH_TIES, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, queryissue.JSON_QUERY_FUNCTIONS_NAME, "", queryissue.JSON_QUERY_FUNCTION, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, queryissue.JSON_CONSTRUCTOR_FUNCTION_NAME, "", queryissue.JSON_CONSTRUCTOR_FUNCTION, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, queryissue.AGGREGATION_FUNCTIONS_NAME, "", queryissue.AGGREGATE_FUNCTION, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, queryissue.SECURITY_INVOKER_VIEWS_NAME, "", queryissue.SECURITY_INVOKER_VIEWS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, queryissue.JSONB_SUBSCRIPTING_NAME, "", queryissue.JSONB_SUBSCRIPTING, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, queryissue.FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME, "", queryissue.FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, queryissue.JSON_TYPE_PREDICATE_NAME, "", queryissue.JSON_TYPE_PREDICATE, schemaAnalysisReport, false, "")) return lo.Filter(unsupportedFeatures, func(f UnsupportedFeature, _ int) bool { return len(f.Objects) > 0 @@ -1079,7 +1095,7 @@ func getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisReport utils.Schema unsupportedIndexDatatypes = append(unsupportedIndexDatatypes, "array") // adding it here only as we know issue form analyze will come with type unsupportedIndexDatatypes = append(unsupportedIndexDatatypes, "user_defined_type") // adding it here as we UDTs will come with this type. for _, unsupportedType := range unsupportedIndexDatatypes { - indexes := getUnsupportedFeaturesFromSchemaAnalysisReport(fmt.Sprintf("%s indexes", unsupportedType), fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, unsupportedType), "", schemaAnalysisReport, false, "") + indexes := getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, fmt.Sprintf("%s indexes", unsupportedType), fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, unsupportedType), "", schemaAnalysisReport, false, "") for _, object := range indexes.Objects { formattedObject := object formattedObject.ObjectName = fmt.Sprintf("%s: %s", strings.ToUpper(unsupportedType), object.ObjectName) @@ -1095,7 +1111,7 @@ func getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisReport utils.Schema func fetchUnsupportedOracleFeaturesFromSchemaReport(schemaAnalysisReport utils.SchemaReport) ([]UnsupportedFeature, error) { log.Infof("fetching unsupported features for Oracle...") unsupportedFeatures := make([]UnsupportedFeature, 0) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(COMPOUND_TRIGGER_FEATURE, COMPOUND_TRIGGER_ISSUE_REASON, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, COMPOUND_TRIGGER_FEATURE, COMPOUND_TRIGGER_ISSUE_REASON, "", schemaAnalysisReport, false, "")) return lo.Filter(unsupportedFeatures, func(f UnsupportedFeature, _ int) bool { return len(f.Objects) > 0 }), nil @@ -1132,13 +1148,41 @@ func fetchUnsupportedObjectTypes() ([]UnsupportedFeature, error) { unsupportedIndexes = append(unsupportedIndexes, ObjectInfo{ ObjectName: fmt.Sprintf("Index Name: %s, Index Type=%s", objectName, objectType), }) + + assessmentReport.AppendIssues(AssessmentIssue{ + Category: constants.FEATURE, + Type: "", // TODO + Name: UNSUPPORTED_INDEXES_FEATURE, + ObjectType: "INDEX", + ObjectName: fmt.Sprintf("Index Name: %s, Index Type=%s", objectName, objectType), + }) } else if objectType == VIRTUAL_COLUMN { virtualColumns = append(virtualColumns, ObjectInfo{ObjectName: objectName}) + assessmentReport.AppendIssues(AssessmentIssue{ + Category: constants.FEATURE, + Type: "", // TODO + Name: VIRTUAL_COLUMNS_FEATURE, + ObjectName: objectName, + }) } else if objectType == INHERITED_TYPE { inheritedTypes = append(inheritedTypes, ObjectInfo{ObjectName: objectName}) + assessmentReport.AppendIssues(AssessmentIssue{ + Category: constants.FEATURE, + Type: "", // TODO + Name: INHERITED_TYPES_FEATURE, + ObjectName: objectName, + }) } else if objectType == REFERENCE_PARTITION || objectType == SYSTEM_PARTITION { referenceOrTablePartitionPresent = true unsupportedPartitionTypes = append(unsupportedPartitionTypes, ObjectInfo{ObjectName: fmt.Sprintf("Table Name: %s, Partition Method: %s", objectName, objectType)}) + + assessmentReport.AppendIssues(AssessmentIssue{ + Category: constants.FEATURE, + Type: "", // TODO + Name: UNSUPPORTED_PARTITIONING_METHODS_FEATURE, + ObjectType: "TABLE", + ObjectName: fmt.Sprintf("Table Name: %s, Partition Method: %s", objectName, objectType), + }) } } @@ -1185,6 +1229,18 @@ func fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport utils.SchemaReport) []U SqlStatement: issue.SqlStatement, }) docsLink = issue.DocsLink + + assessmentReport.AppendIssues(AssessmentIssue{ + Category: constants.PLPGSQL_OBJECT, + Type: issue.Type, + Name: reason, + Impact: issue.Impact, // TODO: verify(expected already there since underlying issues are assigned) + ObjectType: issue.ObjectType, + ObjectName: issue.ObjectName, + SqlStatement: issue.SqlStatement, + DocsLink: issue.DocsLink, + MinimumVersionFixedIn: issue.MinimumVersionsFixedIn, + }) } feature := UnsupportedFeature{ FeatureName: reason, @@ -1260,6 +1316,16 @@ func fetchUnsupportedQueryConstructs() ([]utils.UnsupportedQueryConstruct, error MinimumVersionsFixedIn: issue.MinimumVersionsFixedIn, } result = append(result, uqc) + + assessmentReport.AppendIssues(AssessmentIssue{ + Category: constants.QUERY_CONSTRUCT, + Type: issue.Type, + Name: issue.Name, + Impact: issue.Impact, + SqlStatement: issue.SqlStatement, + DocsLink: issue.DocsLink, + MinimumVersionFixedIn: issue.MinimumVersionsFixedIn, + }) } } @@ -1352,6 +1418,27 @@ func fetchColumnsWithUnsupportedDataTypes() ([]utils.TableColumnsDataTypes, []ut return unsupportedDataTypes, unsupportedDataTypesForLiveMigration, unsupportedDataTypesForLiveMigrationWithFForFB, nil } +func getAssessmentIssuesForUnsupportedDatatypes(unsupportedDatatypes []utils.TableColumnsDataTypes) []AssessmentIssue { + var assessmentIssues []AssessmentIssue + for _, colInfo := range unsupportedDatatypes { + qualifiedColName := fmt.Sprintf("%s.%s.%s", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName) + issue := AssessmentIssue{ + Category: constants.DATATYPE, + CategoryDescription: GetCategoryDescription(constants.DATATYPE), + Type: colInfo.DataType, // TODO: maybe name it like "unsupported datatype - geometry" + Name: colInfo.DataType, // TODO: maybe name it like "unsupported datatype - geometry" + Impact: constants.IMPACT_LEVEL_3, + ObjectType: constants.COLUMN, + ObjectName: qualifiedColName, + DocsLink: "", // TODO + MinimumVersionFixedIn: nil, // TODO + } + assessmentIssues = append(assessmentIssues, issue) + } + + return assessmentIssues +} + /* Queries to ignore: - Collected schemas is totally different than source schema list, not containing "" @@ -1440,17 +1527,29 @@ func addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration case POSTGRESQL: log.Infof("add migration caveats to assessment report") migrationCaveats := make([]UnsupportedFeature, 0) - migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(ALTER_PARTITION_ADD_PK_CAVEAT_FEATURE, "", queryissue.ALTER_TABLE_ADD_PK_ON_PARTITIONED_TABLE, + migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.MIGRATION_CAVEATS, ALTER_PARTITION_ADD_PK_CAVEAT_FEATURE, "", queryissue.ALTER_TABLE_ADD_PK_ON_PARTITIONED_TABLE, schemaAnalysisReport, true, DESCRIPTION_ADD_PK_TO_PARTITION_TABLE)) - migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(FOREIGN_TABLE_CAVEAT_FEATURE, "", queryissue.FOREIGN_TABLE, + migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.MIGRATION_CAVEATS, FOREIGN_TABLE_CAVEAT_FEATURE, "", queryissue.FOREIGN_TABLE, schemaAnalysisReport, false, DESCRIPTION_FOREIGN_TABLES)) - migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(POLICIES_CAVEAT_FEATURE, "", queryissue.POLICY_WITH_ROLES, + migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.MIGRATION_CAVEATS, POLICIES_CAVEAT_FEATURE, "", queryissue.POLICY_WITH_ROLES, schemaAnalysisReport, false, DESCRIPTION_POLICY_ROLE_DESCRIPTION)) if len(unsupportedDataTypesForLiveMigration) > 0 { columns := make([]ObjectInfo, 0) - for _, col := range unsupportedDataTypesForLiveMigration { - columns = append(columns, ObjectInfo{ObjectName: fmt.Sprintf("%s.%s.%s (%s)", col.SchemaName, col.TableName, col.ColumnName, col.DataType)}) + for _, colInfo := range unsupportedDataTypesForLiveMigration { + columns = append(columns, ObjectInfo{ObjectName: fmt.Sprintf("%s.%s.%s (%s)", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName, colInfo.DataType)}) + + assessmentReport.AppendIssues(AssessmentIssue{ + Category: constants.MIGRATION_CAVEATS, + CategoryDescription: "", // TODO + Type: UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE, // TODO add object type in type name + Name: "", // TODO + Impact: constants.IMPACT_LEVEL_1, // Caveat - we don't know the migration is offline/online; + Description: UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_DESCRIPTION, + ObjectType: constants.COLUMN, + ObjectName: fmt.Sprintf("%s.%s.%s", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName), + DocsLink: UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, + }) } if len(columns) > 0 { migrationCaveats = append(migrationCaveats, UnsupportedFeature{UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE, columns, false, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_DESCRIPTION, nil}) @@ -1458,8 +1557,20 @@ func addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration } if len(unsupportedDataTypesForLiveMigrationWithFForFB) > 0 { columns := make([]ObjectInfo, 0) - for _, col := range unsupportedDataTypesForLiveMigrationWithFForFB { - columns = append(columns, ObjectInfo{ObjectName: fmt.Sprintf("%s.%s.%s (%s)", col.SchemaName, col.TableName, col.ColumnName, col.DataType)}) + for _, colInfo := range unsupportedDataTypesForLiveMigrationWithFForFB { + columns = append(columns, ObjectInfo{ObjectName: fmt.Sprintf("%s.%s.%s (%s)", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName, colInfo.DataType)}) + + assessmentReport.AppendIssues(AssessmentIssue{ + Category: constants.MIGRATION_CAVEATS, + CategoryDescription: "", // TODO + Type: UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE, // TODO add object type in type name + Name: "", // TODO + Impact: constants.IMPACT_LEVEL_1, + Description: UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_DESCRIPTION, + ObjectType: constants.COLUMN, + ObjectName: fmt.Sprintf("%s.%s.%s", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName), + DocsLink: UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, + }) } if len(columns) > 0 { migrationCaveats = append(migrationCaveats, UnsupportedFeature{UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE, columns, false, UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_DESCRIPTION, nil}) diff --git a/yb-voyager/cmd/assessMigrationCommandV2.go b/yb-voyager/cmd/assessMigrationCommandV2.go deleted file mode 100644 index 15c0c6bc3..000000000 --- a/yb-voyager/cmd/assessMigrationCommandV2.go +++ /dev/null @@ -1,437 +0,0 @@ -package cmd - -import ( - "fmt" - "strings" - - "github.com/samber/lo" - log "github.com/sirupsen/logrus" - "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" - "github.com/yugabyte/yb-voyager/yb-voyager/src/migassessment" - "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryissue" - "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryparser" - "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" - "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" - "golang.org/x/exp/slices" -) - -// TODO: content of this function will move to generateAssessmentReport() -// For now this function will fetch all the issues which will be used by the migration complexity determination logic -func fetchAllAssessmentIssues() ([]AssessmentIssue, error) { - var assessmentIssues []AssessmentIssue - - assessmentIssues = append(assessmentIssues, getAssessmentReportContentFromAnalyzeSchemaV2()...) - - issues, err := fetchUnsupportedObjectTypesV2() - if err != nil { - return assessmentIssues, fmt.Errorf("failed to fetch unsupported object type issues: %w", err) - } - assessmentIssues = append(assessmentIssues, issues...) - - if utils.GetEnvAsBool("REPORT_UNSUPPORTED_QUERY_CONSTRUCTS", true) { - issues, err := fetchUnsupportedQueryConstructsV2() - if err != nil { - return assessmentIssues, fmt.Errorf("failed to fetch unsupported queries on YugabyteDB: %w", err) - } - assessmentIssues = append(assessmentIssues, issues...) - } - - unsupportedDataTypes, unsupportedDataTypesForLiveMigration, unsupportedDataTypesForLiveMigrationWithFForFB, err := fetchColumnsWithUnsupportedDataTypes() - if err != nil { - return assessmentIssues, fmt.Errorf("failed to fetch columns with unsupported data types: %w", err) - } - - assessmentIssues = append(assessmentIssues, getAssessmentIssuesForUnsupportedDatatypes(unsupportedDataTypes)...) - - assessmentIssues = append(assessmentIssues, fetchMigrationCaveatAssessmentIssues(unsupportedDataTypesForLiveMigration, unsupportedDataTypesForLiveMigrationWithFForFB)...) - - return assessmentIssues, nil -} - -// TODO: replacement of getAssessmentReportContentFromAnalyzeSchema() -func getAssessmentReportContentFromAnalyzeSchemaV2() []AssessmentIssue { - /* - Here we are generating analyze schema report which converts issue instance to analyze schema issue - Then in assessment codepath we extract the required information from analyze schema issue which could have been done directly from issue instance(TODO) - - But current Limitation is analyze schema currently uses regexp etc to detect some issues(not using parser). - */ - schemaAnalysisReport := analyzeSchemaInternal(&source, true) - assessmentReport.MigrationComplexity = schemaAnalysisReport.MigrationComplexity - assessmentReport.SchemaSummary = schemaAnalysisReport.SchemaSummary - assessmentReport.SchemaSummary.Description = lo.Ternary(source.DBType == ORACLE, SCHEMA_SUMMARY_DESCRIPTION_ORACLE, SCHEMA_SUMMARY_DESCRIPTION) - - var assessmentIssues []AssessmentIssue - switch source.DBType { - case ORACLE: - assessmentIssues = append(assessmentIssues, fetchUnsupportedOracleFeaturesFromSchemaReportV2(schemaAnalysisReport)...) - case POSTGRESQL: - assessmentIssues = append(assessmentIssues, fetchUnsupportedPGFeaturesFromSchemaReportV2(schemaAnalysisReport)...) - default: - panic(fmt.Sprintf("unsupported source db type %q", source.DBType)) - } - - // Ques: Do we still need this and REPORT_UNSUPPORTED_QUERY_CONSTRUCTS env var - if utils.GetEnvAsBool("REPORT_UNSUPPORTED_PLPGSQL_OBJECTS", true) { - assessmentIssues = append(assessmentIssues, fetchUnsupportedPlPgSQLObjectsV2(schemaAnalysisReport)...) - } - - return assessmentIssues -} - -// TODO: will replace fetchUnsupportedOracleFeaturesFromSchemaReport() -func fetchUnsupportedOracleFeaturesFromSchemaReportV2(schemaAnalysisReport utils.SchemaReport) []AssessmentIssue { - log.Infof("fetching assessment issues of feature category for Oracle...") - var assessmentIssues []AssessmentIssue - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, "", COMPOUND_TRIGGER_ISSUE_REASON, schemaAnalysisReport, "")...) - return assessmentIssues -} - -// TODO: will replace fetchUnsupportedPGFeaturesFromSchemaReport() -func fetchUnsupportedPGFeaturesFromSchemaReportV2(schemaAnalysisReport utils.SchemaReport) []AssessmentIssue { - log.Infof("fetching assessment issues of feature category for PG...") - - var assessmentIssues []AssessmentIssue - for _, indexMethod := range queryissue.UnsupportedIndexMethods { - displayIndexMethod := strings.ToUpper(indexMethod) - reason := fmt.Sprintf(INDEX_METHOD_ISSUE_REASON, displayIndexMethod) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, "", reason, schemaAnalysisReport, "")...) - } - - assessmentIssues = append(assessmentIssues, getIndexesOnComplexTypeUnsupportedFeatureV2(schemaAnalysisReport, queryissue.UnsupportedIndexDatatypes)...) - - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.CONSTRAINT_TRIGGER, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.INHERITANCE, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.STORED_GENERATED_COLUMNS, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, "", CONVERSION_ISSUE_REASON, schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.MULTI_COLUMN_GIN_INDEX, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.ALTER_TABLE_SET_COLUMN_ATTRIBUTE, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.ALTER_TABLE_DISABLE_RULE, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.ALTER_TABLE_CLUSTER_ON, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.STORAGE_PARAMETER, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, "", UNSUPPORTED_EXTENSION_ISSUE, schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.EXCLUSION_CONSTRAINTS, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.DEFERRABLE_CONSTRAINTS, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, "", VIEW_CHECK_OPTION_ISSUE, schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.PK_UK_ON_COMPLEX_DATATYPE, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.UNLOGGED_TABLE, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.REFERENCING_CLAUSE_IN_TRIGGER, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.BEFORE_ROW_TRIGGER_ON_PARTITIONED_TABLE, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.ADVISORY_LOCKS, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.XML_FUNCTIONS, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.SYSTEM_COLUMNS, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.LARGE_OBJECT_FUNCTIONS, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.REGEX_FUNCTIONS, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.FETCH_WITH_TIES, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.JSON_QUERY_FUNCTION, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.JSON_CONSTRUCTOR_FUNCTION, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.AGGREGATE_FUNCTION, "", schemaAnalysisReport, "")...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, queryissue.SECURITY_INVOKER_VIEWS, "", schemaAnalysisReport, "")...) - - return assessmentIssues -} - -func getIndexesOnComplexTypeUnsupportedFeatureV2(schemaAnalysisReport utils.SchemaReport, unsupportedIndexDatatypes []string) []AssessmentIssue { - var assessmentIssues []AssessmentIssue - // TODO: include MinimumVersionsFixedIn - - unsupportedIndexDatatypes = append(unsupportedIndexDatatypes, "array") // adding it here only as we know issue form analyze will come with type - unsupportedIndexDatatypes = append(unsupportedIndexDatatypes, "user_defined_type") // adding it here as we UDTs will come with this type. - for _, unsupportedType := range unsupportedIndexDatatypes { - // formattedObject.ObjectName = fmt.Sprintf("%s: %s", strings.ToUpper(unsupportedType), object.ObjectName) - issueReason := fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, unsupportedType) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.FEATURE, "", issueReason, schemaAnalysisReport, "")...) - } - return assessmentIssues -} - -// TODO: replacement of fetchUnsupportedObjectTypes() -func fetchUnsupportedObjectTypesV2() ([]AssessmentIssue, error) { - if source.DBType != ORACLE { - return nil, nil - } - - query := fmt.Sprintf(`SELECT schema_name, object_name, object_type FROM %s`, migassessment.OBJECT_TYPE_MAPPING) - rows, err := assessmentDB.Query(query) - if err != nil { - return nil, fmt.Errorf("error querying-%s: %w", query, err) - } - defer func() { - closeErr := rows.Close() - if closeErr != nil { - log.Warnf("error closing rows while fetching object type mapping metadata: %v", err) - } - }() - - var assessmentIssues []AssessmentIssue - for rows.Next() { - var schemaName, objectName, objectType string - err = rows.Scan(&schemaName, &objectName, &objectType) - if err != nil { - return nil, fmt.Errorf("error scanning rows:%w", err) - } - - // For these oracle issues defining Impact not required, since oracle migration complexity comes from ora2pg - switch { - case slices.Contains(OracleUnsupportedIndexTypes, objectType): - assessmentIssues = append(assessmentIssues, AssessmentIssue{ - Category: constants.FEATURE, - Type: "", // TODO - Name: UNSUPPORTED_INDEXES_FEATURE, - ObjectType: "INDEX", - ObjectName: fmt.Sprintf("Index Name: %s, Index Type=%s", objectName, objectType), - }) - case objectType == VIRTUAL_COLUMN: - assessmentIssues = append(assessmentIssues, AssessmentIssue{ - Category: constants.FEATURE, - Type: "", // TODO - Name: VIRTUAL_COLUMNS_FEATURE, - ObjectName: objectName, - }) - case objectType == INHERITED_TYPE: - assessmentIssues = append(assessmentIssues, AssessmentIssue{ - Category: constants.FEATURE, - Type: "", // TODO - Name: INHERITED_TYPES_FEATURE, - ObjectName: objectName, - }) - case objectType == REFERENCE_PARTITION || objectType == SYSTEM_PARTITION: - referenceOrTablePartitionPresent = true - assessmentIssues = append(assessmentIssues, AssessmentIssue{ - Category: constants.FEATURE, - Type: "", // TODO - Name: UNSUPPORTED_PARTITIONING_METHODS_FEATURE, - ObjectType: "TABLE", - ObjectName: fmt.Sprintf("Table Name: %s, Partition Method: %s", objectName, objectType), - }) - } - } - - return assessmentIssues, nil -} - -// // TODO: replacement of fetchUnsupportedPlPgSQLObjects() -func fetchUnsupportedPlPgSQLObjectsV2(schemaAnalysisReport utils.SchemaReport) []AssessmentIssue { - if source.DBType != POSTGRESQL { - return nil - } - - plpgsqlIssues := lo.Filter(schemaAnalysisReport.Issues, func(issue utils.AnalyzeSchemaIssue, _ int) bool { - return issue.IssueType == UNSUPPORTED_PLPGSQL_OBJECTS - }) - groupedPlpgsqlIssuesByReason := lo.GroupBy(plpgsqlIssues, func(issue utils.AnalyzeSchemaIssue) string { - return issue.Reason - }) - - var assessmentIssues []AssessmentIssue - for reason, issues := range groupedPlpgsqlIssuesByReason { - var minVersionsFixedIn map[string]*ybversion.YBVersion - var minVersionsFixedInSet bool - - for _, issue := range issues { - if !minVersionsFixedInSet { - minVersionsFixedIn = issue.MinimumVersionsFixedIn - minVersionsFixedInSet = true - } - if !areMinVersionsFixedInEqual(minVersionsFixedIn, issue.MinimumVersionsFixedIn) { - utils.ErrExit("Issues belonging to UnsupportedFeature %s have different minimum versions fixed in: %v, %v", reason, minVersionsFixedIn, issue.MinimumVersionsFixedIn) - } - - assessmentIssues = append(assessmentIssues, AssessmentIssue{ - Category: constants.PLPGSQL_OBJECT, - Type: issue.Type, - Name: reason, - Impact: issue.Impact, // TODO: verify(expected already there since underlying issues are assigned) - ObjectType: issue.ObjectType, - ObjectName: issue.ObjectName, - SqlStatement: issue.SqlStatement, - DocsLink: issue.DocsLink, - MinimumVersionFixedIn: issue.MinimumVersionsFixedIn, - }) - } - } - - return assessmentIssues -} - -// Q: do we no need of displayDDLInHTML in this approach? DDL can always be there for issues in the table. -func getAssessmentIssuesFromSchemaAnalysisReport(category string, issueType string, issueReason string, schemaAnalysisReport utils.SchemaReport, issueDescription string) []AssessmentIssue { - log.Infof("filtering issues for type: %s", issueType) - var issues []AssessmentIssue - var minVersionsFixedIn map[string]*ybversion.YBVersion - var minVersionsFixedInSet bool - for _, analyzeIssue := range schemaAnalysisReport.Issues { - if !slices.Contains([]string{UNSUPPORTED_FEATURES, MIGRATION_CAVEATS}, analyzeIssue.IssueType) { - continue - } - - issueMatched := lo.Ternary[bool](issueType != "", issueType == analyzeIssue.Type, strings.Contains(analyzeIssue.Reason, issueReason)) - if issueMatched { - if !minVersionsFixedInSet { - minVersionsFixedIn = analyzeIssue.MinimumVersionsFixedIn - minVersionsFixedInSet = true - } - if !areMinVersionsFixedInEqual(minVersionsFixedIn, analyzeIssue.MinimumVersionsFixedIn) { - utils.ErrExit("Issues belonging to type %s have different minimum versions fixed in: %v, %v", analyzeIssue.Type, minVersionsFixedIn, analyzeIssue.MinimumVersionsFixedIn) - } - - issues = append(issues, AssessmentIssue{ - Category: category, - CategoryDescription: GetCategoryDescription(category), - Type: analyzeIssue.Type, - Name: analyzeIssue.Reason, // in convertIssueInstanceToAnalyzeIssue() we assign IssueType to Reason field - Description: issueDescription, // TODO: verify - Impact: analyzeIssue.Impact, - ObjectType: analyzeIssue.ObjectType, - ObjectName: analyzeIssue.ObjectName, - SqlStatement: analyzeIssue.SqlStatement, - DocsLink: analyzeIssue.DocsLink, - MinimumVersionFixedIn: minVersionsFixedIn, - }) - } - } - - return issues -} - -// TODO: soon to replace fetchUnsupportedQueryConstructs() -func fetchUnsupportedQueryConstructsV2() ([]AssessmentIssue, error) { - if source.DBType != POSTGRESQL { - return nil, nil - } - - query := fmt.Sprintf("SELECT DISTINCT query from %s", migassessment.DB_QUERIES_SUMMARY) - rows, err := assessmentDB.Query(query) - if err != nil { - return nil, fmt.Errorf("error querying=%s on assessmentDB: %w", query, err) - } - defer func() { - closeErr := rows.Close() - if closeErr != nil { - log.Warnf("error closing rows while fetching database queries summary metadata: %v", err) - } - }() - - var executedQueries []string - for rows.Next() { - var executedQuery string - err := rows.Scan(&executedQuery) - if err != nil { - return nil, fmt.Errorf("error scanning rows: %w", err) - } - executedQueries = append(executedQueries, executedQuery) - } - - if len(executedQueries) == 0 { - log.Infof("queries info not present in the assessment metadata for detecting unsupported query constructs") - return nil, nil - } - - var assessmentIssues []AssessmentIssue - for i := 0; i < len(executedQueries); i++ { - query := executedQueries[i] - log.Debugf("fetching unsupported query constructs for query - [%s]", query) - collectedSchemaList, err := queryparser.GetSchemaUsed(query) - if err != nil { // no need to error out if failed to get schemas for a query - log.Errorf("failed to get schemas used for query [%s]: %v", query, err) - continue - } - - log.Infof("collected schema list %v(len=%d) for query [%s]", collectedSchemaList, len(collectedSchemaList), query) - if !considerQueryForIssueDetection(collectedSchemaList) { - log.Infof("ignoring query due to difference in collected schema list %v(len=%d) vs source schema list %v(len=%d)", - collectedSchemaList, len(collectedSchemaList), source.GetSchemaList(), len(source.GetSchemaList())) - continue - } - - issues, err := parserIssueDetector.GetDMLIssues(query, targetDbVersion) - if err != nil { - log.Errorf("failed while trying to fetch query issues in query - [%s]: %v", - query, err) - } - - for _, issue := range issues { - issue := AssessmentIssue{ - Category: constants.QUERY_CONSTRUCT, - Type: issue.Type, - Name: issue.Name, - Impact: issue.Impact, - SqlStatement: issue.SqlStatement, - DocsLink: issue.DocsLink, - MinimumVersionFixedIn: issue.MinimumVersionsFixedIn, - } - assessmentIssues = append(assessmentIssues, issue) - } - } - - return assessmentIssues, nil -} - -func getAssessmentIssuesForUnsupportedDatatypes(unsupportedDatatypes []utils.TableColumnsDataTypes) []AssessmentIssue { - var assessmentIssues []AssessmentIssue - for _, colInfo := range unsupportedDatatypes { - qualifiedColName := fmt.Sprintf("%s.%s.%s", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName) - issue := AssessmentIssue{ - Category: constants.DATATYPE, - CategoryDescription: GetCategoryDescription(constants.DATATYPE), - Type: colInfo.DataType, // TODO: maybe name it like "unsupported datatype - geometry" - Name: colInfo.DataType, // TODO: maybe name it like "unsupported datatype - geometry" - Impact: constants.IMPACT_LEVEL_3, - ObjectType: constants.COLUMN, - ObjectName: qualifiedColName, - DocsLink: "", // TODO - MinimumVersionFixedIn: nil, // TODO - } - assessmentIssues = append(assessmentIssues, issue) - } - - return assessmentIssues -} - -// TODO: soon to replace addMigrationCaveatsToAssessmentReport() -func fetchMigrationCaveatAssessmentIssues(unsupportedDataTypesForLiveMigration []utils.TableColumnsDataTypes, unsupportedDataTypesForLiveMigrationWithFForFB []utils.TableColumnsDataTypes) []AssessmentIssue { - var assessmentIssues []AssessmentIssue - switch source.DBType { - case POSTGRESQL: - log.Infof("fetching migration caveat category assessment issues") - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.MIGRATION_CAVEATS, queryissue.ALTER_TABLE_ADD_PK_ON_PARTITIONED_TABLE, "", schemaAnalysisReport, DESCRIPTION_ADD_PK_TO_PARTITION_TABLE)...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.MIGRATION_CAVEATS, queryissue.FOREIGN_TABLE, "", schemaAnalysisReport, DESCRIPTION_FOREIGN_TABLES)...) - assessmentIssues = append(assessmentIssues, getAssessmentIssuesFromSchemaAnalysisReport(constants.MIGRATION_CAVEATS, queryissue.POLICY_WITH_ROLES, "", schemaAnalysisReport, DESCRIPTION_POLICY_ROLE_DESCRIPTION)...) - - if len(unsupportedDataTypesForLiveMigration) > 0 { - for _, colInfo := range unsupportedDataTypesForLiveMigration { - issue := AssessmentIssue{ - Category: constants.MIGRATION_CAVEATS, - CategoryDescription: "", // TODO - Type: UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE, // TODO add object type in type name - Name: "", // TODO - Impact: constants.IMPACT_LEVEL_1, // Caveat - we don't know the migration is offline/online; - Description: UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_DESCRIPTION, - ObjectType: constants.COLUMN, - ObjectName: fmt.Sprintf("%s.%s.%s", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName), - DocsLink: UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, - } - assessmentIssues = append(assessmentIssues, issue) - } - } - - if len(unsupportedDataTypesForLiveMigrationWithFForFB) > 0 { - for _, colInfo := range unsupportedDataTypesForLiveMigrationWithFForFB { - issue := AssessmentIssue{ - Category: constants.MIGRATION_CAVEATS, - CategoryDescription: "", // TODO - Type: UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE, // TODO add object type in type name - Name: "", // TODO - Impact: constants.IMPACT_LEVEL_1, - Description: UNSUPPORTED_DATATYPES_FOR_LIVE_MIGRATION_WITH_FF_FB_DESCRIPTION, - ObjectType: constants.COLUMN, - ObjectName: fmt.Sprintf("%s.%s.%s", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName), - DocsLink: UNSUPPORTED_DATATYPE_LIVE_MIGRATION_DOC_LINK, - } - assessmentIssues = append(assessmentIssues, issue) - } - } - } - - return assessmentIssues -} diff --git a/yb-voyager/cmd/common.go b/yb-voyager/cmd/common.go index 064d80262..2542c14d1 100644 --- a/yb-voyager/cmd/common.go +++ b/yb-voyager/cmd/common.go @@ -1315,6 +1315,10 @@ func ParseJSONToAssessmentReport(reportPath string) (*AssessmentReport, error) { return &report, nil } +func (ar *AssessmentReport) AppendIssues(issues ...AssessmentIssue) { + ar.Issues = append(ar.Issues, issues...) +} + func (ar *AssessmentReport) GetShardedTablesRecommendation() ([]string, error) { if ar.Sizing == nil { return nil, fmt.Errorf("sizing report is null, can't fetch sharded tables") diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index c1ac35092..269a07caf 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -80,12 +80,12 @@ func NewRegexFunctionsIssue(objectType string, objectName string, sqlStatement s } var aggregateFunctionIssue = issue.Issue{ - Type: AGGREGATE_FUNCTION, + Type: AGGREGATE_FUNCTION, Name: AGGREGATION_FUNCTIONS_NAME, Description: "any_value, range_agg and range_intersect_agg functions not supported yet in YugabyteDB", - Suggestion: "", - GH: "", - DocsLink: "", + Suggestion: "", + GH: "", + DocsLink: "", } func NewAggregationFunctionIssue(objectType string, objectName string, sqlStatement string, funcNames []string) QueryIssue { @@ -148,12 +148,12 @@ func NewLOFuntionsIssue(objectType string, objectName string, sqlStatement strin } var jsonbSubscriptingIssue = issue.Issue{ - Type: JSONB_SUBSCRIPTING, - TypeName: JSONB_SUBSCRIPTING_NAME, - TypeDescription: "Jsonb subscripting is not supported in YugabyteDB yet", - Suggestion: "Use Arrow operators (-> / ->>) to access the jsonb fields.", - GH: "", - DocsLink: "", //TODO + Type: JSONB_SUBSCRIPTING, + Name: JSONB_SUBSCRIPTING_NAME, + Description: "Jsonb subscripting is not supported in YugabyteDB yet", + Suggestion: "Use Arrow operators (-> / ->>) to access the jsonb fields.", + GH: "", + DocsLink: "", //TODO } func NewJsonbSubscriptingIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -161,12 +161,12 @@ func NewJsonbSubscriptingIssue(objectType string, objectName string, sqlStatemen } var jsonPredicateIssue = issue.Issue{ - Type: JSON_TYPE_PREDICATE, + Type: JSON_TYPE_PREDICATE, Name: JSON_TYPE_PREDICATE_NAME, Description: "IS JSON predicate expressions not supported yet in YugabyteDB", - Suggestion: "", - GH: "", - DocsLink: "", //TODO + Suggestion: "", + GH: "", + DocsLink: "", //TODO } func NewJsonPredicateIssue(objectType string, objectName string, sqlStatement string) QueryIssue { From 3ca100cbb3f1001eb004d1d13dfd9102cf48a680 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Tue, 7 Jan 2025 16:50:28 +0000 Subject: [PATCH 14/16] added impact for issues missed before --- yb-voyager/cmd/assessMigrationCommand.go | 1 + yb-voyager/src/query/queryissue/issues_ddl.go | 1 + yb-voyager/src/query/queryissue/issues_dml.go | 6 ++++++ 3 files changed, 8 insertions(+) diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index c5463d134..90b3cb70f 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -1176,6 +1176,7 @@ func fetchUnsupportedObjectTypes() ([]UnsupportedFeature, error) { referenceOrTablePartitionPresent = true unsupportedPartitionTypes = append(unsupportedPartitionTypes, ObjectInfo{ObjectName: fmt.Sprintf("Table Name: %s, Partition Method: %s", objectName, objectType)}) + // For oracle migration complexity comes from ora2pg, so defining Impact not required right now assessmentReport.AppendIssues(AssessmentIssue{ Category: constants.FEATURE, Type: "", // TODO diff --git a/yb-voyager/src/query/queryissue/issues_ddl.go b/yb-voyager/src/query/queryissue/issues_ddl.go index 1187ec810..7a06bcaa0 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl.go +++ b/yb-voyager/src/query/queryissue/issues_ddl.go @@ -503,6 +503,7 @@ func NewSecurityInvokerViewIssue(objectType string, objectName string, SqlStatem var foreignKeyReferencesPartitionedTableIssue = issue.Issue{ Type: FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE, Name: FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME, + Impact: constants.IMPACT_LEVEL_1, Suggestion: "No workaround available ", GH: "", // TODO DocsLink: "", // TODO diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index 269a07caf..5250d2683 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -82,6 +82,7 @@ func NewRegexFunctionsIssue(objectType string, objectName string, sqlStatement s var aggregateFunctionIssue = issue.Issue{ Type: AGGREGATE_FUNCTION, Name: AGGREGATION_FUNCTIONS_NAME, + Impact: constants.IMPACT_LEVEL_2, Description: "any_value, range_agg and range_intersect_agg functions not supported yet in YugabyteDB", Suggestion: "", GH: "", @@ -99,6 +100,7 @@ func NewAggregationFunctionIssue(objectType string, objectName string, sqlStatem var jsonConstructorFunctionsIssue = issue.Issue{ Type: JSON_CONSTRUCTOR_FUNCTION, Name: JSON_CONSTRUCTOR_FUNCTION_NAME, + Impact: constants.IMPACT_LEVEL_2, Description: "Postgresql 17 features not supported yet in YugabyteDB", Suggestion: "", GH: "", @@ -116,6 +118,7 @@ func NewJsonConstructorFunctionIssue(objectType string, objectName string, sqlSt var jsonQueryFunctionIssue = issue.Issue{ Type: JSON_QUERY_FUNCTION, Name: JSON_QUERY_FUNCTIONS_NAME, + Impact: constants.IMPACT_LEVEL_2, Description: "Postgresql 17 features not supported yet in YugabyteDB", Suggestion: "", GH: "", @@ -133,6 +136,7 @@ func NewJsonQueryFunctionIssue(objectType string, objectName string, sqlStatemen var loFunctionsIssue = issue.Issue{ Type: LARGE_OBJECT_FUNCTIONS, Name: LARGE_OBJECT_FUNCTIONS_NAME, + Impact: constants.IMPACT_LEVEL_2, Description: "Large Objects functions are not supported in YugabyteDB", Suggestion: "Large objects functions are not yet supported in YugabyteDB, no workaround available right now", GH: "https://github.com/yugabyte/yugabyte-db/issues/25318", @@ -163,6 +167,7 @@ func NewJsonbSubscriptingIssue(objectType string, objectName string, sqlStatemen var jsonPredicateIssue = issue.Issue{ Type: JSON_TYPE_PREDICATE, Name: JSON_TYPE_PREDICATE_NAME, + Impact: constants.IMPACT_LEVEL_2, Description: "IS JSON predicate expressions not supported yet in YugabyteDB", Suggestion: "", GH: "", @@ -204,6 +209,7 @@ func NewCopyOnErrorIssue(objectType string, objectName string, sqlStatement stri var fetchWithTiesIssue = issue.Issue{ Type: FETCH_WITH_TIES, Name: "FETCH .. WITH TIES", + Impact: constants.IMPACT_LEVEL_2, Description: "FETCH .. WITH TIES is not supported in YugabyteDB", Suggestion: "No workaround available right now", GH: "", From afe39a0a532a32b02aa69831db9a19ec45095c8c Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Wed, 8 Jan 2025 05:56:32 +0000 Subject: [PATCH 15/16] changes as per review comments --- yb-voyager/cmd/analyzeSchema.go | 2 +- yb-voyager/src/issue/issue.go | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 79efec3b8..55ea06e1c 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -678,7 +678,7 @@ func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fil ObjectName: displayObjectName, Reason: issueInstance.Name, Type: issueInstance.Type, - Impact: issueInstance.GetImpact(), + Impact: issueInstance.Impact, SqlStatement: issueInstance.SqlStatement, DocsLink: issueInstance.DocsLink, FilePath: fileName, diff --git a/yb-voyager/src/issue/issue.go b/yb-voyager/src/issue/issue.go index f1c9358c2..47bf663fb 100644 --- a/yb-voyager/src/issue/issue.go +++ b/yb-voyager/src/issue/issue.go @@ -17,8 +17,6 @@ limitations under the License. package issue import ( - "github.com/samber/lo" - "github.com/yugabyte/yb-voyager/yb-voyager/src/constants" "github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion" ) @@ -44,11 +42,6 @@ func (i Issue) IsFixedIn(v *ybversion.YBVersion) (bool, error) { return v.GreaterThanOrEqual(minVersionFixedInSeries), nil } -func (i Issue) GetImpact() string { - // Default value as IMPACT_LEVEL_1 if not defined in issue - return lo.Ternary(i.Impact != "", i.Impact, constants.IMPACT_LEVEL_1) -} - /* Dynamic Impact Determination (TODO) - We can define the impact calculator function based on issue type wherever/whenever needed @@ -72,7 +65,6 @@ func (i Issue) GetImpact() string { return "LEVEL_3" } - // Update existing GetImpact() method func (i Issue) GetImpact(stats *PgStats) string { if calc, ok := impactCalculators[i.Type]; ok { return calc(i, stats) From b7968ab732987d70aabd38d2ad4cf21b0c6b2c00 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Wed, 8 Jan 2025 08:08:21 +0000 Subject: [PATCH 16/16] review comments, renames and cleanup --- yb-voyager/cmd/analyzeSchema.go | 118 ++++++++-------- yb-voyager/cmd/assessMigrationCommand.go | 128 ++++++++---------- yb-voyager/cmd/constants.go | 23 ++-- yb-voyager/src/constants/constants.go | 2 +- yb-voyager/src/query/queryissue/issues_dml.go | 1 + .../src/query/queryparser/object_collector.go | 2 +- 6 files changed, 133 insertions(+), 141 deletions(-) diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index 55ea06e1c..3eb78dc3b 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -221,7 +221,7 @@ const ( ) // Reports one case in JSON -func reportCase(filePath string, reason string, ghIssue string, suggestion string, objType string, objName string, sqlStmt string, issueType string, docsLink string, impact string) { +func reportCase(filePath string, reason string, ghIssue string, suggestion string, objType string, objName string, sqlStmt string, category string, docsLink string, impact string) { var issue utils.AnalyzeSchemaIssue issue.FilePath = filePath issue.Reason = reason @@ -230,7 +230,7 @@ func reportCase(filePath string, reason string, ghIssue string, suggestion strin issue.ObjectType = objType issue.ObjectName = objName issue.SqlStatement = sqlStmt - issue.IssueType = issueType + issue.IssueType = category // IssueType field of analyze schema should be renamed to Category issue.Impact = lo.Ternary(impact != "", impact, constants.IMPACT_LEVEL_1) if sourceDBType == POSTGRESQL { issue.DocsLink = docsLink @@ -241,13 +241,13 @@ func reportCase(filePath string, reason string, ghIssue string, suggestion strin func reportBasedOnComment(comment int, fpath string, issue string, suggestion string, objName string, objType string, line string) { if comment == 1 { - reportCase(fpath, "Unsupported, please edit to match PostgreSQL syntax", "https://github.com/yugabyte/yb-voyager/issues/1625", suggestion, objType, objName, line, UNSUPPORTED_FEATURES, "", "") + reportCase(fpath, "Unsupported, please edit to match PostgreSQL syntax", "https://github.com/yugabyte/yb-voyager/issues/1625", suggestion, objType, objName, line, UNSUPPORTED_FEATURES_CATEGORY, "", "") summaryMap[objType].invalidCount[objName] = true } else if comment == 2 { // reportCase(fpath, "PACKAGE in oracle are exported as Schema, please review and edit to match PostgreSQL syntax if required, Package is "+objName, issue, suggestion, objType, "") summaryMap["PACKAGE"].objSet = append(summaryMap["PACKAGE"].objSet, objName) } else if comment == 3 { - reportCase(fpath, "SQLs in file might be unsupported please review and edit to match PostgreSQL syntax if required. ", "https://github.com/yugabyte/yb-voyager/issues/1625", suggestion, objType, objName, line, UNSUPPORTED_FEATURES, "", "") + reportCase(fpath, "SQLs in file might be unsupported please review and edit to match PostgreSQL syntax if required. ", "https://github.com/yugabyte/yb-voyager/issues/1625", suggestion, objType, objName, line, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if comment == 4 { summaryMap[objType].details["Inherited Types are present which are not supported in PostgreSQL syntax, so exported as Inherited Tables"] = true } @@ -323,7 +323,7 @@ func checkStmtsUsingParser(sqlInfoArr []sqlInfo, fpath string, objType string) { if !summaryMap[objType].invalidCount[sqlStmtInfo.objName] { reason := fmt.Sprintf("%s - '%s'", UNSUPPORTED_PG_SYNTAX, err.Error()) reportCase(fpath, reason, "https://github.com/yugabyte/yb-voyager/issues/1625", - "Fix the schema as per PG syntax", objType, sqlStmtInfo.objName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "Fix the schema as per PG syntax", objType, sqlStmtInfo.objName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } continue } @@ -350,7 +350,7 @@ func checkViews(sqlInfoArr []sqlInfo, fpath string) { if view := viewWithCheckRegex.FindStringSubmatch(sqlInfo.stmt); view != nil { summaryMap["VIEW"].invalidCount[sqlInfo.objName] = true reportCase(fpath, VIEW_CHECK_OPTION_ISSUE, "https://github.com/yugabyte/yugabyte-db/issues/22716", - "Use Trigger with INSTEAD OF clause on INSERT/UPDATE on view to get this functionality", "VIEW", view[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, VIEW_CHECK_OPTION_DOC_LINK, "") + "Use Trigger with INSTEAD OF clause on INSERT/UPDATE on view to get this functionality", "VIEW", view[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, VIEW_CHECK_OPTION_DOC_LINK, "") } } } @@ -374,47 +374,47 @@ func checkSql(sqlInfoArr []sqlInfo, fpath string) { if rangeRegex.MatchString(sqlInfo.stmt) { reportCase(fpath, "RANGE with offset PRECEDING/FOLLOWING is not supported for column type numeric and offset type double precision", - "https://github.com/yugabyte/yugabyte-db/issues/10692", "", "TABLE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/yugabyte/yugabyte-db/issues/10692", "", "TABLE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true } else if stmt := fetchRegex.FindStringSubmatch(sqlInfo.stmt); stmt != nil { location := strings.ToUpper(stmt[1]) if slices.Contains(notSupportedFetchLocation, location) { summaryMap["PROCEDURE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "This FETCH clause might not be supported yet", "https://github.com/YugaByte/yugabyte-db/issues/6514", - "Please verify the DDL on your YugabyteDB version before proceeding", "CURSOR", sqlInfo.objName, sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "Please verify the DDL on your YugabyteDB version before proceeding", "CURSOR", sqlInfo.objName, sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } } else if stmt := alterAggRegex.FindStringSubmatch(sqlInfo.stmt); stmt != nil { summaryMap["AGGREGATE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER AGGREGATE not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/2717", "", "AGGREGATE", stmt[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/2717", "", "AGGREGATE", stmt[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if dropCollRegex.MatchString(sqlInfo.stmt) { summaryMap["COLLATION"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP COLLATION", sqlInfo.formattedStmt), "COLLATION", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP COLLATION", sqlInfo.formattedStmt), "COLLATION", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if dropIdxRegex.MatchString(sqlInfo.stmt) { summaryMap["INDEX"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP INDEX", sqlInfo.formattedStmt), "INDEX", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP INDEX", sqlInfo.formattedStmt), "INDEX", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if dropViewRegex.MatchString(sqlInfo.stmt) { summaryMap["VIEW"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP VIEW", sqlInfo.formattedStmt), "VIEW", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP VIEW", sqlInfo.formattedStmt), "VIEW", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if dropSeqRegex.MatchString(sqlInfo.stmt) { summaryMap["SEQUENCE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP SEQUENCE", sqlInfo.formattedStmt), "SEQUENCE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP SEQUENCE", sqlInfo.formattedStmt), "SEQUENCE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if dropForeignRegex.MatchString(sqlInfo.stmt) { summaryMap["FOREIGN TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP multiple objects not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP FOREIGN TABLE", sqlInfo.formattedStmt), "FOREIGN TABLE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/880", separateMultiObj("DROP FOREIGN TABLE", sqlInfo.formattedStmt), "FOREIGN TABLE", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if idx := dropIdxConcurRegex.FindStringSubmatch(sqlInfo.stmt); idx != nil { summaryMap["INDEX"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "DROP INDEX CONCURRENTLY not supported yet", - "https://github.com/yugabyte/yugabyte-db/issues/22717", "", "INDEX", idx[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/yugabyte/yugabyte-db/issues/22717", "", "INDEX", idx[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if currentOfRegex.MatchString(sqlInfo.stmt) { - reportCase(fpath, "WHERE CURRENT OF not supported yet", "https://github.com/YugaByte/yugabyte-db/issues/737", "", "CURSOR", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + reportCase(fpath, "WHERE CURRENT OF not supported yet", "https://github.com/YugaByte/yugabyte-db/issues/737", "", "CURSOR", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if bulkCollectRegex.MatchString(sqlInfo.stmt) { - reportCase(fpath, "BULK COLLECT keyword of oracle is not converted into PostgreSQL compatible syntax", "https://github.com/yugabyte/yb-voyager/issues/1539", "", "", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + reportCase(fpath, "BULK COLLECT keyword of oracle is not converted into PostgreSQL compatible syntax", "https://github.com/yugabyte/yb-voyager/issues/1539", "", "", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } } } @@ -426,106 +426,106 @@ func checkDDL(sqlInfoArr []sqlInfo, fpath string, objType string) { if am := amRegex.FindStringSubmatch(sqlInfo.stmt); am != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "CREATE ACCESS METHOD is not supported.", - "https://github.com/yugabyte/yugabyte-db/issues/10693", "", "ACCESS METHOD", am[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/yugabyte/yugabyte-db/issues/10693", "", "ACCESS METHOD", am[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := idxConcRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "REINDEX is not supported.", - "https://github.com/yugabyte/yugabyte-db/issues/10267", "", "TABLE", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/yugabyte/yugabyte-db/issues/10267", "", "TABLE", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := likeAllRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "LIKE ALL is not supported yet.", - "https://github.com/yugabyte/yugabyte-db/issues/10697", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/yugabyte/yugabyte-db/issues/10697", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := likeRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "LIKE clause not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1129", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/1129", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := withOidsRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "OIDs are not supported for user tables.", - "https://github.com/yugabyte/yugabyte-db/issues/10273", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/yugabyte/yugabyte-db/issues/10273", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterOfRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE OF not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterSchemaRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE SET SCHEMA not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/3947", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/3947", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if createSchemaRegex.MatchString(sqlInfo.stmt) { summaryMap["SCHEMA"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "CREATE SCHEMA with elements not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/10865", "", "SCHEMA", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/10865", "", "SCHEMA", "", sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterNotOfRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE NOT OF not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterColumnStatsRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE ALTER column SET STATISTICS not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterColumnStorageRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE ALTER column SET STORAGE not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterColumnResetAttributesRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE ALTER column RESET (attribute) not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterConstrRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE ALTER CONSTRAINT not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := setOidsRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE SET WITH OIDS not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[4], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[4], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := withoutClusterRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE SET WITHOUT CLUSTER not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterSetRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE SET not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterIdxRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER INDEX SET not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "INDEX", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "INDEX", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterResetRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE RESET not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterOptionsRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if typ := dropAttrRegex.FindStringSubmatch(sqlInfo.stmt); typ != nil { summaryMap["TYPE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TYPE DROP ATTRIBUTE not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1893", "", "TYPE", typ[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/1893", "", "TYPE", typ[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if typ := alterTypeRegex.FindStringSubmatch(sqlInfo.stmt); typ != nil { summaryMap["TYPE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TYPE not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1893", "", "TYPE", typ[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/1893", "", "TYPE", typ[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := alterInhRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE INHERIT not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := valConstrRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLE VALIDATE CONSTRAINT not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/1124", "", "TABLE", tbl[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if spc := alterTblSpcRegex.FindStringSubmatch(sqlInfo.stmt); spc != nil { summaryMap["TABLESPACE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER TABLESPACE not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1153", "", "TABLESPACE", spc[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/1153", "", "TABLESPACE", spc[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if spc := alterViewRegex.FindStringSubmatch(sqlInfo.stmt); spc != nil { summaryMap["VIEW"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "ALTER VIEW not supported yet.", - "https://github.com/YugaByte/yugabyte-db/issues/1131", "", "VIEW", spc[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/YugaByte/yugabyte-db/issues/1131", "", "VIEW", spc[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := cLangRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { reportCase(fpath, "LANGUAGE C not supported yet.", - "https://github.com/yugabyte/yb-voyager/issues/1540", "", "FUNCTION", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/yugabyte/yb-voyager/issues/1540", "", "FUNCTION", tbl[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") summaryMap["FUNCTION"].invalidCount[sqlInfo.objName] = true } else if strings.Contains(strings.ToLower(sqlInfo.stmt), "drop temporary table") { filePath := strings.Split(fpath, "/") @@ -533,22 +533,22 @@ func checkDDL(sqlInfoArr []sqlInfo, fpath string, objType string) { objType := strings.ToUpper(strings.Split(fileName, ".")[0]) summaryMap[objType].invalidCount[sqlInfo.objName] = true reportCase(fpath, `temporary table is not a supported clause for drop`, - "https://github.com/yugabyte/yb-voyager/issues/705", `remove "temporary" and change it to "drop table"`, objType, sqlInfo.objName, sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, DROP_TEMP_TABLE_DOC_LINK, "") + "https://github.com/yugabyte/yb-voyager/issues/705", `remove "temporary" and change it to "drop table"`, objType, sqlInfo.objName, sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, DROP_TEMP_TABLE_DOC_LINK, "") } else if regMatch := anydataRegex.FindStringSubmatch(sqlInfo.stmt); regMatch != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true - reportCase(fpath, "AnyData datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with AnyData datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + reportCase(fpath, "AnyData datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with AnyData datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if regMatch := anydatasetRegex.FindStringSubmatch(sqlInfo.stmt); regMatch != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true - reportCase(fpath, "AnyDataSet datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with AnyDataSet datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + reportCase(fpath, "AnyDataSet datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with AnyDataSet datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if regMatch := anyTypeRegex.FindStringSubmatch(sqlInfo.stmt); regMatch != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true - reportCase(fpath, "AnyType datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with AnyType datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + reportCase(fpath, "AnyType datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with AnyType datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if regMatch := uriTypeRegex.FindStringSubmatch(sqlInfo.stmt); regMatch != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true - reportCase(fpath, "URIType datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with URIType datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + reportCase(fpath, "URIType datatype doesn't have a mapping in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1541", `Remove the column with URIType datatype or change it to a relevant supported datatype`, "TABLE", regMatch[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if regMatch := jsonFuncRegex.FindStringSubmatch(sqlInfo.stmt); regMatch != nil { summaryMap[objType].invalidCount[sqlInfo.objName] = true - reportCase(fpath, "JSON_ARRAYAGG() function is not available in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1542", `Rename the function to YugabyteDB's equivalent JSON_AGG()`, objType, regMatch[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + reportCase(fpath, "JSON_ARRAYAGG() function is not available in YugabyteDB", "https://github.com/yugabyte/yb-voyager/issues/1542", `Rename the function to YugabyteDB's equivalent JSON_AGG()`, objType, regMatch[3], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } } @@ -561,11 +561,11 @@ func checkForeign(sqlInfoArr []sqlInfo, fpath string) { if tbl := primRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "Primary key constraints are not supported on foreign tables.", - "https://github.com/yugabyte/yugabyte-db/issues/10698", "", "TABLE", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/yugabyte/yugabyte-db/issues/10698", "", "TABLE", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } else if tbl := foreignKeyRegex.FindStringSubmatch(sqlInfo.stmt); tbl != nil { summaryMap["TABLE"].invalidCount[sqlInfo.objName] = true reportCase(fpath, "Foreign key constraints are not supported on foreign tables.", - "https://github.com/yugabyte/yugabyte-db/issues/10699", "", "TABLE", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/yugabyte/yugabyte-db/issues/10699", "", "TABLE", tbl[1], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") } } } @@ -575,7 +575,7 @@ func checkRemaining(sqlInfoArr []sqlInfo, fpath string) { for _, sqlInfo := range sqlInfoArr { if trig := compoundTrigRegex.FindStringSubmatch(sqlInfo.stmt); trig != nil { reportCase(fpath, COMPOUND_TRIGGER_ISSUE_REASON, - "https://github.com/yugabyte/yb-voyager/issues/1543", "", "TRIGGER", trig[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, "", "") + "https://github.com/yugabyte/yb-voyager/issues/1543", "", "TRIGGER", trig[2], sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, "", "") summaryMap["TRIGGER"].invalidCount[sqlInfo.objName] = true } } @@ -622,18 +622,18 @@ var MigrationCaveatsIssues = []string{ } func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fileName string, isPlPgSQLIssue bool) utils.AnalyzeSchemaIssue { - issueType := UNSUPPORTED_FEATURES + issueType := UNSUPPORTED_FEATURES_CATEGORY switch true { case isPlPgSQLIssue: - issueType = UNSUPPORTED_PLPGSQL_OBJECTS + issueType = UNSUPPORTED_PLPGSQL_OBJECTS_CATEGORY case slices.ContainsFunc(MigrationCaveatsIssues, func(i string) bool { //Adding the MIGRATION_CAVEATS issueType of the utils.Issue for these issueInstances in MigrationCaveatsIssues return strings.Contains(issueInstance.Name, i) }): - issueType = MIGRATION_CAVEATS + issueType = MIGRATION_CAVEATS_CATEGORY case strings.HasPrefix(issueInstance.Name, UNSUPPORTED_DATATYPE): //Adding the UNSUPPORTED_DATATYPES issueType of the utils.Issue for these issues whose TypeName starts with "Unsupported datatype ..." - issueType = UNSUPPORTED_DATATYPES + issueType = UNSUPPORTED_DATATYPES_CATEGORY } var constraintIssues = []string{ @@ -693,7 +693,7 @@ func checkExtensions(sqlInfoArr []sqlInfo, fpath string) { if sqlInfo.objName != "" && !slices.Contains(supportedExtensionsOnYB, sqlInfo.objName) { summaryMap["EXTENSION"].invalidCount[sqlInfo.objName] = true reportCase(fpath, UNSUPPORTED_EXTENSION_ISSUE+" Refer to the docs link for the more information on supported extensions.", "https://github.com/yugabyte/yb-voyager/issues/1538", "", "EXTENSION", - sqlInfo.objName, sqlInfo.formattedStmt, UNSUPPORTED_FEATURES, EXTENSION_DOC_LINK, constants.IMPACT_LEVEL_3) + sqlInfo.objName, sqlInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, EXTENSION_DOC_LINK, constants.IMPACT_LEVEL_3) } if strings.ToLower(sqlInfo.objName) == "hll" { summaryMap["EXTENSION"].details[`'hll' extension is supported in YugabyteDB v2.18 onwards. Please verify this extension as per the target YugabyteDB version.`] = true @@ -1113,12 +1113,12 @@ func checkConversions(sqlInfoArr []sqlInfo, filePath string) { convName = fmt.Sprintf("%s.%s", convName, nameList[1].GetString_().Sval) } reportCase(filePath, CONVERSION_ISSUE_REASON, "https://github.com/yugabyte/yugabyte-db/issues/10866", - "Remove it from the exported schema", "CONVERSION", convName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, CREATE_CONVERSION_DOC_LINK, constants.IMPACT_LEVEL_3) + "Remove it from the exported schema", "CONVERSION", convName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, CREATE_CONVERSION_DOC_LINK, constants.IMPACT_LEVEL_3) } else { //pg_query doesn't seem to have a Node type of AlterConversionStmt so using regex for now if stmt := alterConvRegex.FindStringSubmatch(sqlStmtInfo.stmt); stmt != nil { reportCase(filePath, "ALTER CONVERSION is not supported yet", "https://github.com/YugaByte/yugabyte-db/issues/10866", - "Remove it from the exported schema", "CONVERSION", stmt[1], sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, CREATE_CONVERSION_DOC_LINK, "") + "Remove it from the exported schema", "CONVERSION", stmt[1], sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES_CATEGORY, CREATE_CONVERSION_DOC_LINK, "") } } diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 90b3cb70f..a80de6b4e 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -425,16 +425,6 @@ func assessMigration() (err error) { } log.Infof("number of assessment issues detected: %d\n", len(assessmentReport.Issues)) - // var data string - // for i, issue := range assessmentReport.Issues { - // data += fmt.Sprintf("AssessmentIssue[%d] = %s\n\n", i, spew.Sdump(issue)) - // } - // assessIssuesFile := filepath.Join(exportDir, "assessment_issues.txt") - // err = os.WriteFile(assessIssuesFile, []byte(data), 0677) - // if err != nil { - // panic("failed to write to assessment_issues.txt") - // } - utils.PrintAndLog("Migration assessment completed successfully.") completedEvent := createMigrationAssessmentCompletedEvent() controlPlane.MigrationAssessmentCompleted(completedEvent) @@ -551,7 +541,7 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment for _, unsupportedDataType := range ar.UnsupportedDataTypes { issues = append(issues, AssessmentIssueYugabyteD{ Type: constants.DATATYPE, - TypeDescription: DATATYPE_CATEGORY_DESCRIPTION, + TypeDescription: GetCategoryDescription(constants.DATATYPE), Subtype: unsupportedDataType.DataType, ObjectName: fmt.Sprintf("%s.%s.%s", unsupportedDataType.SchemaName, unsupportedDataType.TableName, unsupportedDataType.ColumnName), SqlStatement: "", @@ -563,7 +553,7 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment for _, object := range unsupportedFeature.Objects { issues = append(issues, AssessmentIssueYugabyteD{ Type: constants.FEATURE, - TypeDescription: FEATURE_CATEGORY_DESCRIPTION, + TypeDescription: GetCategoryDescription(constants.FEATURE), Subtype: unsupportedFeature.FeatureName, SubtypeDescription: unsupportedFeature.FeatureDescription, // TODO: test payload once we add desc for unsupported features ObjectName: object.ObjectName, @@ -578,7 +568,7 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment for _, object := range migrationCaveat.Objects { issues = append(issues, AssessmentIssueYugabyteD{ Type: constants.MIGRATION_CAVEATS, - TypeDescription: MIGRATION_CAVEATS_CATEGORY_DESCRIPTION, + TypeDescription: GetCategoryDescription(constants.MIGRATION_CAVEATS), Subtype: migrationCaveat.FeatureName, SubtypeDescription: migrationCaveat.FeatureDescription, ObjectName: object.ObjectName, @@ -592,7 +582,7 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment for _, uqc := range ar.UnsupportedQueryConstructs { issues = append(issues, AssessmentIssueYugabyteD{ Type: constants.QUERY_CONSTRUCT, - TypeDescription: UNSUPPORTED_QUERY_CONSTRUCTS_CATEGORY_DESCRIPTION, + TypeDescription: GetCategoryDescription(constants.QUERY_CONSTRUCT), Subtype: uqc.ConstructTypeName, SqlStatement: uqc.Query, DocsLink: uqc.DocsLink, @@ -604,7 +594,7 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment for _, object := range plpgsqlObjects.Objects { issues = append(issues, AssessmentIssueYugabyteD{ Type: constants.PLPGSQL_OBJECT, - TypeDescription: UNSUPPPORTED_PLPGSQL_OBJECT_CATEGORY_DESCRIPTION, + TypeDescription: GetCategoryDescription(constants.PLPGSQL_OBJECT), Subtype: plpgsqlObjects.FeatureName, SubtypeDescription: plpgsqlObjects.FeatureDescription, ObjectName: object.ObjectName, @@ -985,7 +975,7 @@ func areMinVersionsFixedInEqual(m1 map[string]*ybversion.YBVersion, m2 map[strin return true } -func getUnsupportedFeaturesFromSchemaAnalysisReport(category string, featureName string, issueReason string, issueType string, schemaAnalysisReport utils.SchemaReport, displayDDLInHTML bool, description string) UnsupportedFeature { +func getUnsupportedFeaturesFromSchemaAnalysisReport(featureName string, issueReason string, issueType string, schemaAnalysisReport utils.SchemaReport, displayDDLInHTML bool, description string) UnsupportedFeature { log.Info("filtering issues for feature: ", featureName) objects := make([]ObjectInfo, 0) link := "" // for oracle we shouldn't display any line for links @@ -993,7 +983,7 @@ func getUnsupportedFeaturesFromSchemaAnalysisReport(category string, featureName var minVersionsFixedInSet bool for _, analyzeIssue := range schemaAnalysisReport.Issues { - if !slices.Contains([]string{UNSUPPORTED_FEATURES, MIGRATION_CAVEATS}, analyzeIssue.IssueType) { + if !slices.Contains([]string{UNSUPPORTED_FEATURES_CATEGORY, MIGRATION_CAVEATS_CATEGORY}, analyzeIssue.IssueType) { continue } issueMatched := lo.Ternary[bool](issueType != "", issueType == analyzeIssue.Type, strings.Contains(analyzeIssue.Reason, issueReason)) @@ -1014,7 +1004,7 @@ func getUnsupportedFeaturesFromSchemaAnalysisReport(category string, featureName link = analyzeIssue.DocsLink objects = append(objects, objectInfo) - assessmentReport.AppendIssues(convertAnalyzeSchemaIssueToAssessmentIssue(analyzeIssue, category, description, minVersionsFixedIn)) + assessmentReport.AppendIssues(convertAnalyzeSchemaIssueToAssessmentIssue(analyzeIssue, description, minVersionsFixedIn)) } } @@ -1022,10 +1012,10 @@ func getUnsupportedFeaturesFromSchemaAnalysisReport(category string, featureName } // Q: do we no need of displayDDLInHTML in this approach? DDL can always be there for issues in the table if present. -func convertAnalyzeSchemaIssueToAssessmentIssue(analyzeSchemaIssue utils.AnalyzeSchemaIssue, category string, issueDescription string, minVersionsFixedIn map[string]*ybversion.YBVersion) AssessmentIssue { +func convertAnalyzeSchemaIssueToAssessmentIssue(analyzeSchemaIssue utils.AnalyzeSchemaIssue, issueDescription string, minVersionsFixedIn map[string]*ybversion.YBVersion) AssessmentIssue { return AssessmentIssue{ - Category: category, - CategoryDescription: GetCategoryDescription(category), + Category: analyzeSchemaIssue.IssueType, + CategoryDescription: GetCategoryDescription(analyzeSchemaIssue.IssueType), Type: analyzeSchemaIssue.Type, Name: analyzeSchemaIssue.Reason, // in convertIssueInstanceToAnalyzeIssue() we assign IssueType to Reason field Description: issueDescription, // TODO: verify @@ -1046,39 +1036,39 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem displayIndexMethod := strings.ToUpper(indexMethod) featureName := fmt.Sprintf("%s indexes", displayIndexMethod) reason := fmt.Sprintf(INDEX_METHOD_ISSUE_REASON, displayIndexMethod) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, featureName, reason, "", schemaAnalysisReport, false, "")) - } - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, CONSTRAINT_TRIGGERS_FEATURE, "", queryissue.CONSTRAINT_TRIGGER, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, INHERITED_TABLES_FEATURE, "", queryissue.INHERITANCE, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, GENERATED_COLUMNS_FEATURE, "", queryissue.STORED_GENERATED_COLUMNS, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, CONVERSIONS_OBJECTS_FEATURE, CONVERSION_ISSUE_REASON, "", schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, MULTI_COLUMN_GIN_INDEX_FEATURE, "", queryissue.MULTI_COLUMN_GIN_INDEX, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, ALTER_SETTING_ATTRIBUTE_FEATURE, "", queryissue.ALTER_TABLE_SET_COLUMN_ATTRIBUTE, schemaAnalysisReport, true, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, DISABLING_TABLE_RULE_FEATURE, "", queryissue.ALTER_TABLE_DISABLE_RULE, schemaAnalysisReport, true, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, CLUSTER_ON_FEATURE, "", queryissue.ALTER_TABLE_CLUSTER_ON, schemaAnalysisReport, true, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, STORAGE_PARAMETERS_FEATURE, "", queryissue.STORAGE_PARAMETER, schemaAnalysisReport, true, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, EXTENSION_FEATURE, UNSUPPORTED_EXTENSION_ISSUE, "", schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, EXCLUSION_CONSTRAINT_FEATURE, "", queryissue.EXCLUSION_CONSTRAINTS, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, DEFERRABLE_CONSTRAINT_FEATURE, "", queryissue.DEFERRABLE_CONSTRAINTS, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, VIEW_CHECK_FEATURE, VIEW_CHECK_OPTION_ISSUE, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(featureName, reason, "", schemaAnalysisReport, false, "")) + } + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CONSTRAINT_TRIGGERS_FEATURE, "", queryissue.CONSTRAINT_TRIGGER, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(INHERITED_TABLES_FEATURE, "", queryissue.INHERITANCE, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(GENERATED_COLUMNS_FEATURE, "", queryissue.STORED_GENERATED_COLUMNS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CONVERSIONS_OBJECTS_FEATURE, CONVERSION_ISSUE_REASON, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(MULTI_COLUMN_GIN_INDEX_FEATURE, "", queryissue.MULTI_COLUMN_GIN_INDEX, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(ALTER_SETTING_ATTRIBUTE_FEATURE, "", queryissue.ALTER_TABLE_SET_COLUMN_ATTRIBUTE, schemaAnalysisReport, true, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(DISABLING_TABLE_RULE_FEATURE, "", queryissue.ALTER_TABLE_DISABLE_RULE, schemaAnalysisReport, true, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(CLUSTER_ON_FEATURE, "", queryissue.ALTER_TABLE_CLUSTER_ON, schemaAnalysisReport, true, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(STORAGE_PARAMETERS_FEATURE, "", queryissue.STORAGE_PARAMETER, schemaAnalysisReport, true, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(EXTENSION_FEATURE, UNSUPPORTED_EXTENSION_ISSUE, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(EXCLUSION_CONSTRAINT_FEATURE, "", queryissue.EXCLUSION_CONSTRAINTS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(DEFERRABLE_CONSTRAINT_FEATURE, "", queryissue.DEFERRABLE_CONSTRAINTS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(VIEW_CHECK_FEATURE, VIEW_CHECK_OPTION_ISSUE, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisReport, queryissue.UnsupportedIndexDatatypes)) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, PK_UK_CONSTRAINT_ON_COMPLEX_DATATYPES_FEATURE, "", queryissue.PK_UK_ON_COMPLEX_DATATYPE, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, UNLOGGED_TABLE_FEATURE, "", queryissue.UNLOGGED_TABLE, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, REFERENCING_TRIGGER_FEATURE, "", queryissue.REFERENCING_CLAUSE_IN_TRIGGER, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE_FEATURE, "", queryissue.BEFORE_ROW_TRIGGER_ON_PARTITIONED_TABLE, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, queryissue.ADVISORY_LOCKS_NAME, "", queryissue.ADVISORY_LOCKS, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, queryissue.XML_FUNCTIONS_NAME, "", queryissue.XML_FUNCTIONS, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, queryissue.SYSTEM_COLUMNS_NAME, "", queryissue.SYSTEM_COLUMNS, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, queryissue.LARGE_OBJECT_FUNCTIONS_NAME, "", queryissue.LARGE_OBJECT_FUNCTIONS, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, REGEX_FUNCTIONS_FEATURE, "", queryissue.REGEX_FUNCTIONS, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, FETCH_WITH_TIES_FEATURE, "", queryissue.FETCH_WITH_TIES, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, queryissue.JSON_QUERY_FUNCTIONS_NAME, "", queryissue.JSON_QUERY_FUNCTION, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, queryissue.JSON_CONSTRUCTOR_FUNCTION_NAME, "", queryissue.JSON_CONSTRUCTOR_FUNCTION, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, queryissue.AGGREGATION_FUNCTIONS_NAME, "", queryissue.AGGREGATE_FUNCTION, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, queryissue.SECURITY_INVOKER_VIEWS_NAME, "", queryissue.SECURITY_INVOKER_VIEWS, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, queryissue.JSONB_SUBSCRIPTING_NAME, "", queryissue.JSONB_SUBSCRIPTING, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, queryissue.FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME, "", queryissue.FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE, schemaAnalysisReport, false, "")) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, queryissue.JSON_TYPE_PREDICATE_NAME, "", queryissue.JSON_TYPE_PREDICATE, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(PK_UK_CONSTRAINT_ON_COMPLEX_DATATYPES_FEATURE, "", queryissue.PK_UK_ON_COMPLEX_DATATYPE, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(UNLOGGED_TABLE_FEATURE, "", queryissue.UNLOGGED_TABLE, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REFERENCING_TRIGGER_FEATURE, "", queryissue.REFERENCING_CLAUSE_IN_TRIGGER, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE_FEATURE, "", queryissue.BEFORE_ROW_TRIGGER_ON_PARTITIONED_TABLE, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.ADVISORY_LOCKS_NAME, "", queryissue.ADVISORY_LOCKS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.XML_FUNCTIONS_NAME, "", queryissue.XML_FUNCTIONS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SYSTEM_COLUMNS_NAME, "", queryissue.SYSTEM_COLUMNS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.LARGE_OBJECT_FUNCTIONS_NAME, "", queryissue.LARGE_OBJECT_FUNCTIONS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REGEX_FUNCTIONS_FEATURE, "", queryissue.REGEX_FUNCTIONS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(FETCH_WITH_TIES_FEATURE, "", queryissue.FETCH_WITH_TIES, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_QUERY_FUNCTIONS_NAME, "", queryissue.JSON_QUERY_FUNCTION, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_CONSTRUCTOR_FUNCTION_NAME, "", queryissue.JSON_CONSTRUCTOR_FUNCTION, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.AGGREGATION_FUNCTIONS_NAME, "", queryissue.AGGREGATE_FUNCTION, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SECURITY_INVOKER_VIEWS_NAME, "", queryissue.SECURITY_INVOKER_VIEWS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSONB_SUBSCRIPTING_NAME, "", queryissue.JSONB_SUBSCRIPTING, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME, "", queryissue.FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_TYPE_PREDICATE_NAME, "", queryissue.JSON_TYPE_PREDICATE, schemaAnalysisReport, false, "")) return lo.Filter(unsupportedFeatures, func(f UnsupportedFeature, _ int) bool { return len(f.Objects) > 0 @@ -1095,7 +1085,7 @@ func getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisReport utils.Schema unsupportedIndexDatatypes = append(unsupportedIndexDatatypes, "array") // adding it here only as we know issue form analyze will come with type unsupportedIndexDatatypes = append(unsupportedIndexDatatypes, "user_defined_type") // adding it here as we UDTs will come with this type. for _, unsupportedType := range unsupportedIndexDatatypes { - indexes := getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, fmt.Sprintf("%s indexes", unsupportedType), fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, unsupportedType), "", schemaAnalysisReport, false, "") + indexes := getUnsupportedFeaturesFromSchemaAnalysisReport(fmt.Sprintf("%s indexes", unsupportedType), fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, unsupportedType), "", schemaAnalysisReport, false, "") for _, object := range indexes.Objects { formattedObject := object formattedObject.ObjectName = fmt.Sprintf("%s: %s", strings.ToUpper(unsupportedType), object.ObjectName) @@ -1111,7 +1101,7 @@ func getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisReport utils.Schema func fetchUnsupportedOracleFeaturesFromSchemaReport(schemaAnalysisReport utils.SchemaReport) ([]UnsupportedFeature, error) { log.Infof("fetching unsupported features for Oracle...") unsupportedFeatures := make([]UnsupportedFeature, 0) - unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.FEATURE, COMPOUND_TRIGGER_FEATURE, COMPOUND_TRIGGER_ISSUE_REASON, "", schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(COMPOUND_TRIGGER_FEATURE, COMPOUND_TRIGGER_ISSUE_REASON, "", schemaAnalysisReport, false, "")) return lo.Filter(unsupportedFeatures, func(f UnsupportedFeature, _ int) bool { return len(f.Objects) > 0 }), nil @@ -1150,7 +1140,7 @@ func fetchUnsupportedObjectTypes() ([]UnsupportedFeature, error) { }) assessmentReport.AppendIssues(AssessmentIssue{ - Category: constants.FEATURE, + Category: UNSUPPORTED_FEATURES_CATEGORY, Type: "", // TODO Name: UNSUPPORTED_INDEXES_FEATURE, ObjectType: "INDEX", @@ -1159,7 +1149,7 @@ func fetchUnsupportedObjectTypes() ([]UnsupportedFeature, error) { } else if objectType == VIRTUAL_COLUMN { virtualColumns = append(virtualColumns, ObjectInfo{ObjectName: objectName}) assessmentReport.AppendIssues(AssessmentIssue{ - Category: constants.FEATURE, + Category: UNSUPPORTED_FEATURES_CATEGORY, Type: "", // TODO Name: VIRTUAL_COLUMNS_FEATURE, ObjectName: objectName, @@ -1167,7 +1157,7 @@ func fetchUnsupportedObjectTypes() ([]UnsupportedFeature, error) { } else if objectType == INHERITED_TYPE { inheritedTypes = append(inheritedTypes, ObjectInfo{ObjectName: objectName}) assessmentReport.AppendIssues(AssessmentIssue{ - Category: constants.FEATURE, + Category: UNSUPPORTED_FEATURES_CATEGORY, Type: "", // TODO Name: INHERITED_TYPES_FEATURE, ObjectName: objectName, @@ -1178,7 +1168,7 @@ func fetchUnsupportedObjectTypes() ([]UnsupportedFeature, error) { // For oracle migration complexity comes from ora2pg, so defining Impact not required right now assessmentReport.AppendIssues(AssessmentIssue{ - Category: constants.FEATURE, + Category: UNSUPPORTED_FEATURES_CATEGORY, Type: "", // TODO Name: UNSUPPORTED_PARTITIONING_METHODS_FEATURE, ObjectType: "TABLE", @@ -1203,7 +1193,7 @@ func fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport utils.SchemaReport) []U } plpgsqlIssues := lo.Filter(schemaAnalysisReport.Issues, func(issue utils.AnalyzeSchemaIssue, _ int) bool { - return issue.IssueType == UNSUPPORTED_PLPGSQL_OBJECTS + return issue.IssueType == UNSUPPORTED_PLPGSQL_OBJECTS_CATEGORY }) groupPlpgsqlIssuesByReason := lo.GroupBy(plpgsqlIssues, func(issue utils.AnalyzeSchemaIssue) string { return issue.Reason @@ -1232,7 +1222,7 @@ func fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport utils.SchemaReport) []U docsLink = issue.DocsLink assessmentReport.AppendIssues(AssessmentIssue{ - Category: constants.PLPGSQL_OBJECT, + Category: UNSUPPORTED_PLPGSQL_OBJECTS_CATEGORY, Type: issue.Type, Name: reason, Impact: issue.Impact, // TODO: verify(expected already there since underlying issues are assigned) @@ -1319,7 +1309,7 @@ func fetchUnsupportedQueryConstructs() ([]utils.UnsupportedQueryConstruct, error result = append(result, uqc) assessmentReport.AppendIssues(AssessmentIssue{ - Category: constants.QUERY_CONSTRUCT, + Category: UNSUPPORTED_QUERY_CONSTRUCTS_CATEGORY, Type: issue.Type, Name: issue.Name, Impact: issue.Impact, @@ -1424,8 +1414,8 @@ func getAssessmentIssuesForUnsupportedDatatypes(unsupportedDatatypes []utils.Tab for _, colInfo := range unsupportedDatatypes { qualifiedColName := fmt.Sprintf("%s.%s.%s", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName) issue := AssessmentIssue{ - Category: constants.DATATYPE, - CategoryDescription: GetCategoryDescription(constants.DATATYPE), + Category: UNSUPPORTED_DATATYPES_CATEGORY, + CategoryDescription: GetCategoryDescription(UNSUPPORTED_DATATYPES_CATEGORY), Type: colInfo.DataType, // TODO: maybe name it like "unsupported datatype - geometry" Name: colInfo.DataType, // TODO: maybe name it like "unsupported datatype - geometry" Impact: constants.IMPACT_LEVEL_3, @@ -1528,11 +1518,11 @@ func addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration case POSTGRESQL: log.Infof("add migration caveats to assessment report") migrationCaveats := make([]UnsupportedFeature, 0) - migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.MIGRATION_CAVEATS, ALTER_PARTITION_ADD_PK_CAVEAT_FEATURE, "", queryissue.ALTER_TABLE_ADD_PK_ON_PARTITIONED_TABLE, + migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(ALTER_PARTITION_ADD_PK_CAVEAT_FEATURE, "", queryissue.ALTER_TABLE_ADD_PK_ON_PARTITIONED_TABLE, schemaAnalysisReport, true, DESCRIPTION_ADD_PK_TO_PARTITION_TABLE)) - migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.MIGRATION_CAVEATS, FOREIGN_TABLE_CAVEAT_FEATURE, "", queryissue.FOREIGN_TABLE, + migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(FOREIGN_TABLE_CAVEAT_FEATURE, "", queryissue.FOREIGN_TABLE, schemaAnalysisReport, false, DESCRIPTION_FOREIGN_TABLES)) - migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(constants.MIGRATION_CAVEATS, POLICIES_CAVEAT_FEATURE, "", queryissue.POLICY_WITH_ROLES, + migrationCaveats = append(migrationCaveats, getUnsupportedFeaturesFromSchemaAnalysisReport(POLICIES_CAVEAT_FEATURE, "", queryissue.POLICY_WITH_ROLES, schemaAnalysisReport, false, DESCRIPTION_POLICY_ROLE_DESCRIPTION)) if len(unsupportedDataTypesForLiveMigration) > 0 { @@ -1541,7 +1531,7 @@ func addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration columns = append(columns, ObjectInfo{ObjectName: fmt.Sprintf("%s.%s.%s (%s)", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName, colInfo.DataType)}) assessmentReport.AppendIssues(AssessmentIssue{ - Category: constants.MIGRATION_CAVEATS, + Category: MIGRATION_CAVEATS_CATEGORY, CategoryDescription: "", // TODO Type: UNSUPPORTED_DATATYPES_LIVE_CAVEAT_FEATURE, // TODO add object type in type name Name: "", // TODO @@ -1562,7 +1552,7 @@ func addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration columns = append(columns, ObjectInfo{ObjectName: fmt.Sprintf("%s.%s.%s (%s)", colInfo.SchemaName, colInfo.TableName, colInfo.ColumnName, colInfo.DataType)}) assessmentReport.AppendIssues(AssessmentIssue{ - Category: constants.MIGRATION_CAVEATS, + Category: MIGRATION_CAVEATS_CATEGORY, CategoryDescription: "", // TODO Type: UNSUPPORTED_DATATYPES_LIVE_WITH_FF_FB_CAVEAT_FEATURE, // TODO add object type in type name Name: "", // TODO diff --git a/yb-voyager/cmd/constants.go b/yb-voyager/cmd/constants.go index e1f0ff1a6..5a2d496be 100644 --- a/yb-voyager/cmd/constants.go +++ b/yb-voyager/cmd/constants.go @@ -107,11 +107,12 @@ const ( REFERENCE_PARTITION = "REFERENCE PARTITION" SYSTEM_PARTITION = "SYSTEM PARTITION" - UNSUPPORTED_FEATURES = "unsupported_features" - UNSUPPORTED_DATATYPES = "unsupported_datatypes" - UNSUPPORTED_PLPGSQL_OBJECTS = "unsupported_plpgsql_objects" - MIGRATION_CAVEATS = "migration_caveats" - REPORT_UNSUPPORTED_QUERY_CONSTRUCTS = "REPORT_UNSUPPORTED_QUERY_CONSTRUCTS" + UNSUPPORTED_FEATURES_CATEGORY = "unsupported_features" + UNSUPPORTED_DATATYPES_CATEGORY = "unsupported_datatypes" + UNSUPPORTED_QUERY_CONSTRUCTS_CATEGORY = "unsupported_query_constructs" + UNSUPPORTED_PLPGSQL_OBJECTS_CATEGORY = "unsupported_plpgsql_objects" + MIGRATION_CAVEATS_CATEGORY = "migration_caveats" + REPORT_UNSUPPORTED_QUERY_CONSTRUCTS = "REPORT_UNSUPPORTED_QUERY_CONSTRUCTS" HTML = "html" JSON = "json" @@ -247,18 +248,18 @@ var EVENT_BATCH_MAX_RETRY_COUNT = 50 // returns the description for a given assessment issue category func GetCategoryDescription(category string) string { switch category { - case constants.FEATURE: + case UNSUPPORTED_FEATURES_CATEGORY, constants.FEATURE: return FEATURE_CATEGORY_DESCRIPTION - case constants.DATATYPE: + case UNSUPPORTED_DATATYPES_CATEGORY, constants.DATATYPE: return DATATYPE_CATEGORY_DESCRIPTION - case constants.QUERY_CONSTRUCT: + case UNSUPPORTED_QUERY_CONSTRUCTS_CATEGORY, constants.QUERY_CONSTRUCT: return UNSUPPORTED_QUERY_CONSTRUCTS_CATEGORY_DESCRIPTION - case constants.PLPGSQL_OBJECT: + case UNSUPPORTED_PLPGSQL_OBJECTS_CATEGORY, constants.PLPGSQL_OBJECT: return UNSUPPPORTED_PLPGSQL_OBJECT_CATEGORY_DESCRIPTION - case constants.MIGRATION_CAVEATS: + case MIGRATION_CAVEATS_CATEGORY: // or constants.MIGRATION_CAVEATS (identical) return MIGRATION_CAVEATS_CATEGORY_DESCRIPTION default: - utils.ErrExit("unsupported assessment issue category %q", category) + utils.ErrExit("ERROR: unsupported assessment issue category %q", category) } return "" } diff --git a/yb-voyager/src/constants/constants.go b/yb-voyager/src/constants/constants.go index 809d1a301..9e194afe6 100644 --- a/yb-voyager/src/constants/constants.go +++ b/yb-voyager/src/constants/constants.go @@ -39,4 +39,4 @@ const ( IMPACT_LEVEL_1 = "LEVEL_1" // Represents minimal impact like only the schema ddl IMPACT_LEVEL_2 = "LEVEL_2" // Represents moderate impact like dml queries which might impact a lot of implementation/assumption in app layer IMPACT_LEVEL_3 = "LEVEL_3" // Represent significant impact like TABLE INHERITANCE, which doesn't have any simple workaround but can impact multiple objects/apps -) +) \ No newline at end of file diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index 5250d2683..393841a79 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -154,6 +154,7 @@ func NewLOFuntionsIssue(objectType string, objectName string, sqlStatement strin var jsonbSubscriptingIssue = issue.Issue{ Type: JSONB_SUBSCRIPTING, Name: JSONB_SUBSCRIPTING_NAME, + Impact: constants.IMPACT_LEVEL_2, Description: "Jsonb subscripting is not supported in YugabyteDB yet", Suggestion: "Use Arrow operators (-> / ->>) to access the jsonb fields.", GH: "", diff --git a/yb-voyager/src/query/queryparser/object_collector.go b/yb-voyager/src/query/queryparser/object_collector.go index ba7561a1b..9a6020699 100644 --- a/yb-voyager/src/query/queryparser/object_collector.go +++ b/yb-voyager/src/query/queryparser/object_collector.go @@ -93,7 +93,7 @@ func (c *ObjectCollector) Collect(msg protoreflect.Message) { relName := GetStringField(relationMsg, "relname") objectName := utils.BuildObjectName(schemaName, relName) log.Debugf("[IUD] fetched schemaname=%s relname=%s objectname=%s field\n", schemaName, relName, objectName) - if c.predicate(schemaName, relName, "table") { + if c.predicate(schemaName, relName, constants.TABLE) { c.addObject(objectName) } }