Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Send target-db-version, MinimumVersionsFixedIn details for assessment/analyze-schema reports to controlplane/callhome #2049

Merged
merged 10 commits into from
Dec 12, 2024
+88 −72
Merged

Send target-db-version, MinimumVersionsFixedIn details for assessment/analyze-schema reports to controlplane/callhome #2049

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
45cc90e
assessMigration targetDbVersion
makalaaneesh Dec 9, 2024
1adb3f0
MinimumVersionsFixedIn for assessment issues in control plane
makalaaneesh Dec 9, 2024
2564a24
callhome assessment target db version
makalaaneesh Dec 9, 2024
ea6f6e5
callhome send target db version
makalaaneesh Dec 9, 2024
e17b81d
change from string to *ybversion.YBVersion
makalaaneesh Dec 9, 2024
c836597
DB case
makalaaneesh Dec 9, 2024
dfcff2d
bug fix
makalaaneesh Dec 9, 2024
9f67c36
update upgrade test
makalaaneesh Dec 10, 2024
a2cdfbe
Merge branch 'main' into aneesh/version-issues-callhome-controlplane
makalaaneesh Dec 11, 2024
0931a00
update PayloadVersion
makalaaneesh Dec 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 12 additions & 10 deletions yb-voyager/cmd/analyzeSchema.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -626,15 +626,16 @@ func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fil
}

return utils.Issue{
ObjectType: issueInstance.ObjectType,
ObjectName: issueInstance.ObjectName,
Reason: issueInstance.TypeName,
SqlStatement: issueInstance.SqlStatement,
DocsLink: issueInstance.DocsLink,
FilePath: fileName,
IssueType: issueType,
Suggestion: issueInstance.Suggestion,
GH: issueInstance.GH,
ObjectType: issueInstance.ObjectType,
ObjectName: issueInstance.ObjectName,
Reason: issueInstance.TypeName,
SqlStatement: issueInstance.SqlStatement,
DocsLink: issueInstance.DocsLink,
FilePath: fileName,
IssueType: issueType,
Suggestion: issueInstance.Suggestion,
GH: issueInstance.GH,
MinimumVersionsFixedIn: issueInstance.MinimumVersionsFixedIn,
}
}

Expand Down Expand Up @@ -1217,7 +1218,8 @@ func packAndSendAnalyzeSchemaPayload(status string) {
}

analyzePayload := callhome.AnalyzePhasePayload{
Issues: callhome.MarshalledJsonString(callhomeIssues),
TargetDBVersion: schemaAnalysisReport.TargetDBVersion,
Issues: callhome.MarshalledJsonString(callhomeIssues),
DatabaseObjects: callhome.MarshalledJsonString(lo.Map(schemaAnalysisReport.SchemaSummary.DBObjects, func(dbObject utils.DBObject, _ int) utils.DBObject {
dbObject.ObjectNames = ""
return dbObject
Expand Down
58 changes: 32 additions & 26 deletions yb-voyager/cmd/assessMigrationCommand.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@ func packAndSendAssessMigrationPayload(status string, errMsg string) {
})

assessPayload := callhome.AssessMigrationPhasePayload{
TargetDBVersion: assessmentReport.TargetDBVersion,
MigrationComplexity: assessmentReport.MigrationComplexity,
UnsupportedFeatures: callhome.MarshalledJsonString(lo.Map(assessmentReport.UnsupportedFeatures, func(feature UnsupportedFeature, _ int) callhome.UnsupportedFeature {
var objects []string
Expand Down Expand Up @@ -473,6 +474,7 @@ func createMigrationAssessmentCompletedEvent() *cp.MigrationAssessmentCompletedE
payload := AssessMigrationPayload{
PayloadVersion: ASSESS_MIGRATION_PAYLOAD_VERSION,
VoyagerVersion: assessmentReport.VoyagerVersion,
TargetDBVersion: assessmentReport.TargetDBVersion,
MigrationComplexity: assessmentReport.MigrationComplexity,
SchemaSummary: assessmentReport.SchemaSummary,
AssessmentIssues: assessmentIssues,
Expand Down Expand Up @@ -525,51 +527,55 @@ func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []Assessment
for _, unsupportedFeature := range ar.UnsupportedFeatures {
for _, object := range unsupportedFeature.Objects {
issues = append(issues, AssessmentIssuePayload{
Type: FEATURE,
TypeDescription: FEATURE_ISSUE_TYPE_DESCRIPTION,
Subtype: unsupportedFeature.FeatureName,
SubtypeDescription: unsupportedFeature.FeatureDescription, // TODO: test payload once we add desc for unsupported features
ObjectName: object.ObjectName,
SqlStatement: object.SqlStatement,
DocsLink: unsupportedFeature.DocsLink,
Type: FEATURE,
TypeDescription: FEATURE_ISSUE_TYPE_DESCRIPTION,
Subtype: unsupportedFeature.FeatureName,
SubtypeDescription: unsupportedFeature.FeatureDescription, // TODO: test payload once we add desc for unsupported features
ObjectName: object.ObjectName,
SqlStatement: object.SqlStatement,
DocsLink: unsupportedFeature.DocsLink,
MinimumVersionsFixedIn: unsupportedFeature.MinimumVersionsFixedIn,
})
}
}

for _, migrationCaveat := range ar.MigrationCaveats {
for _, object := range migrationCaveat.Objects {
issues = append(issues, AssessmentIssuePayload{
Type: MIGRATION_CAVEATS,
TypeDescription: MIGRATION_CAVEATS_TYPE_DESCRIPTION,
Subtype: migrationCaveat.FeatureName,
SubtypeDescription: migrationCaveat.FeatureDescription,
ObjectName: object.ObjectName,
SqlStatement: object.SqlStatement,
DocsLink: migrationCaveat.DocsLink,
Type: MIGRATION_CAVEATS,
TypeDescription: MIGRATION_CAVEATS_TYPE_DESCRIPTION,
Subtype: migrationCaveat.FeatureName,
SubtypeDescription: migrationCaveat.FeatureDescription,
ObjectName: object.ObjectName,
SqlStatement: object.SqlStatement,
DocsLink: migrationCaveat.DocsLink,
MinimumVersionsFixedIn: migrationCaveat.MinimumVersionsFixedIn,
})
}
}

for _, uqc := range ar.UnsupportedQueryConstructs {
issues = append(issues, AssessmentIssuePayload{
Type: QUERY_CONSTRUCT,
TypeDescription: UNSUPPORTED_QUERY_CONSTRUTS_DESCRIPTION,
Subtype: uqc.ConstructTypeName,
SqlStatement: uqc.Query,
DocsLink: uqc.DocsLink,
Type: QUERY_CONSTRUCT,
TypeDescription: UNSUPPORTED_QUERY_CONSTRUTS_DESCRIPTION,
Subtype: uqc.ConstructTypeName,
SqlStatement: uqc.Query,
DocsLink: uqc.DocsLink,
MinimumVersionsFixedIn: uqc.MinimumVersionsFixedIn,
})
}

for _, plpgsqlObjects := range ar.UnsupportedPlPgSqlObjects {
for _, object := range plpgsqlObjects.Objects {
issues = append(issues, AssessmentIssuePayload{
Type: PLPGSQL_OBJECT,
TypeDescription: UNSUPPPORTED_PLPGSQL_OBJECT_DESCRIPTION,
Subtype: plpgsqlObjects.FeatureName,
SubtypeDescription: plpgsqlObjects.FeatureDescription,
ObjectName: object.ObjectName,
SqlStatement: object.SqlStatement,
DocsLink: plpgsqlObjects.DocsLink,
Type: PLPGSQL_OBJECT,
TypeDescription: UNSUPPPORTED_PLPGSQL_OBJECT_DESCRIPTION,
Subtype: plpgsqlObjects.FeatureName,
SubtypeDescription: plpgsqlObjects.FeatureDescription,
ObjectName: object.ObjectName,
SqlStatement: object.SqlStatement,
DocsLink: plpgsqlObjects.DocsLink,
MinimumVersionsFixedIn: plpgsqlObjects.MinimumVersionsFixedIn,
})
}
}
Expand Down
18 changes: 10 additions & 8 deletions yb-voyager/cmd/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1224,6 +1224,7 @@ type AssessMigrationDBConfig struct {
type AssessMigrationPayload struct {
PayloadVersion string
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we change the payload version as we add the TargetDBVersion struct?

VoyagerVersion string
TargetDBVersion *ybversion.YBVersion
MigrationComplexity string
SchemaSummary utils.SchemaSummary
AssessmentIssues []AssessmentIssuePayload
Expand All @@ -1235,13 +1236,14 @@ type AssessMigrationPayload struct {
}

type AssessmentIssuePayload struct {
Type string `json:"Type"` // Feature, DataType, MigrationCaveat, UQC
TypeDescription string `json:"TypeDescription"` // Based on AssessmentIssue type
Subtype string `json:"Subtype"` // GIN Indexes, Advisory Locks etc
SubtypeDescription string `json:"SubtypeDescription"` // description based on subtype
ObjectName string `json:"ObjectName"` // Fully qualified object name(empty if NA, eg UQC)
SqlStatement string `json:"SqlStatement"` // DDL or DML(UQC)
DocsLink string `json:"DocsLink"` // docs link based on the subtype
Type string `json:"Type"` // Feature, DataType, MigrationCaveat, UQC
TypeDescription string `json:"TypeDescription"` // Based on AssessmentIssue type
Subtype string `json:"Subtype"` // GIN Indexes, Advisory Locks etc
SubtypeDescription string `json:"SubtypeDescription"` // description based on subtype
ObjectName string `json:"ObjectName"` // Fully qualified object name(empty if NA, eg UQC)
SqlStatement string `json:"SqlStatement"` // DDL or DML(UQC)
DocsLink string `json:"DocsLink"` // docs link based on the subtype
MinimumVersionsFixedIn map[string]*ybversion.YBVersion `json:"MinimumVersionsFixedIn"` // key: series (2024.1, 2.21, etc)

// Store Type-specific details - extensible, can refer any struct
Details json.RawMessage `json:"Details,omitempty"`
Expand All @@ -1268,7 +1270,7 @@ type TargetSizingRecommendations struct {
TotalShardedSize int64
}

var ASSESS_MIGRATION_PAYLOAD_VERSION = "1.0"
var ASSESS_MIGRATION_PAYLOAD_VERSION = "1.1"

//====== AssesmentReport struct methods ======//

Expand Down
31 changes: 17 additions & 14 deletions yb-voyager/src/callhome/diagnostics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
log "github.com/sirupsen/logrus"

"github.com/yugabyte/yb-voyager/yb-voyager/src/utils"
"github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion"
)

// call-home json formats
Expand Down Expand Up @@ -103,18 +104,19 @@ type UnsupportedFeature struct {
}

type AssessMigrationPhasePayload struct {
MigrationComplexity string `json:"migration_complexity"`
UnsupportedFeatures string `json:"unsupported_features"`
UnsupportedDatatypes string `json:"unsupported_datatypes"`
UnsupportedQueryConstructs string `json:"unsupported_query_constructs"`
MigrationCaveats string `json:"migration_caveats"`
UnsupportedPlPgSqlObjects string `json:"unsupported_plpgsql_objects"`
Error string `json:"error,omitempty"` // Removed it for now, TODO
TableSizingStats string `json:"table_sizing_stats"`
IndexSizingStats string `json:"index_sizing_stats"`
SchemaSummary string `json:"schema_summary"`
SourceConnectivity bool `json:"source_connectivity"`
IopsInterval int64 `json:"iops_interval"`
TargetDBVersion *ybversion.YBVersion `json:"target_db_version"`
MigrationComplexity string `json:"migration_complexity"`
UnsupportedFeatures string `json:"unsupported_features"`
UnsupportedDatatypes string `json:"unsupported_datatypes"`
UnsupportedQueryConstructs string `json:"unsupported_query_constructs"`
MigrationCaveats string `json:"migration_caveats"`
UnsupportedPlPgSqlObjects string `json:"unsupported_plpgsql_objects"`
Error string `json:"error,omitempty"` // Removed it for now, TODO
TableSizingStats string `json:"table_sizing_stats"`
IndexSizingStats string `json:"index_sizing_stats"`
SchemaSummary string `json:"schema_summary"`
SourceConnectivity bool `json:"source_connectivity"`
IopsInterval int64 `json:"iops_interval"`
}

type AssessMigrationBulkPhasePayload struct {
Expand All @@ -139,8 +141,9 @@ type ExportSchemaPhasePayload struct {
// SHOULD NOT REMOVE THESE TWO (issues, database_objects) FIELDS of AnalyzePhasePayload as parsing these specifically here
// https://github.com/yugabyte/yugabyte-growth/blob/ad5df306c50c05136df77cd6548a1091ae577046/diagnostics_v2/main.py#L563
type AnalyzePhasePayload struct {
Issues string `json:"issues"`
DatabaseObjects string `json:"database_objects"`
TargetDBVersion *ybversion.YBVersion `json:"target_db_version"`
Issues string `json:"issues"`
DatabaseObjects string `json:"database_objects"`
}
type ExportDataPhasePayload struct {
ParallelJobs int64 `json:"parallel_jobs"`
Expand Down
31 changes: 17 additions & 14 deletions yb-voyager/src/callhome/diagnostics_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (

"github.com/google/uuid"
"github.com/yugabyte/yb-voyager/yb-voyager/src/testutils"
"github.com/yugabyte/yb-voyager/yb-voyager/src/ybversion"
)

func TestCallhomeStructs(t *testing.T) {
Expand Down Expand Up @@ -67,18 +68,19 @@ func TestCallhomeStructs(t *testing.T) {
name: "Validate AssessMigrationPhasePayload Struct Definition",
actualType: reflect.TypeOf(AssessMigrationPhasePayload{}),
expectedType: struct {
MigrationComplexity string `json:"migration_complexity"`
UnsupportedFeatures string `json:"unsupported_features"`
UnsupportedDatatypes string `json:"unsupported_datatypes"`
UnsupportedQueryConstructs string `json:"unsupported_query_constructs"`
MigrationCaveats string `json:"migration_caveats"`
UnsupportedPlPgSqlObjects string `json:"unsupported_plpgsql_objects"`
Error string `json:"error,omitempty"`
TableSizingStats string `json:"table_sizing_stats"`
IndexSizingStats string `json:"index_sizing_stats"`
SchemaSummary string `json:"schema_summary"`
SourceConnectivity bool `json:"source_connectivity"`
IopsInterval int64 `json:"iops_interval"`
TargetDBVersion *ybversion.YBVersion `json:"target_db_version"`
MigrationComplexity string `json:"migration_complexity"`
UnsupportedFeatures string `json:"unsupported_features"`
UnsupportedDatatypes string `json:"unsupported_datatypes"`
UnsupportedQueryConstructs string `json:"unsupported_query_constructs"`
MigrationCaveats string `json:"migration_caveats"`
UnsupportedPlPgSqlObjects string `json:"unsupported_plpgsql_objects"`
Error string `json:"error,omitempty"`
TableSizingStats string `json:"table_sizing_stats"`
IndexSizingStats string `json:"index_sizing_stats"`
SchemaSummary string `json:"schema_summary"`
SourceConnectivity bool `json:"source_connectivity"`
IopsInterval int64 `json:"iops_interval"`
}{},
},
{
Expand Down Expand Up @@ -113,8 +115,9 @@ func TestCallhomeStructs(t *testing.T) {
name: "Validate AnalyzePhasePayload Struct Definition",
actualType: reflect.TypeOf(AnalyzePhasePayload{}),
expectedType: struct {
Issues string `json:"issues"`
DatabaseObjects string `json:"database_objects"`
TargetDBVersion *ybversion.YBVersion `json:"target_db_version"`
Issues string `json:"issues"`
DatabaseObjects string `json:"database_objects"`
}{},
priyanshi-yb marked this conversation as resolved.
Show resolved Hide resolved
},
{
Expand Down