Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reporting Foreign Key references Partitioned table in analyze and assessment #2145

Merged
merged 10 commits into from
Jan 6, 2025
3 changes: 1 addition & 2 deletions yb-voyager/cmd/exportSchema.go
Original file line number Diff line number Diff line change
Expand Up @@ -467,8 +467,7 @@ func applyShardingRecommendationIfMatching(sqlInfo *sqlInfo, shardedTables []str
}

// true -> oracle, false -> PG
parsedObjectName := lo.Ternary(relation.Schemaname == "", relation.Relname,
relation.Schemaname+"."+relation.Relname)
parsedObjectName := utils.GetObjectName(relation.Schemaname, relation.Relname)

match := false
switch source.DBType {
Expand Down
36 changes: 20 additions & 16 deletions yb-voyager/src/query/queryparser/ddl_processor.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import (
pg_query "github.com/pganalyze/pg_query_go/v6"
"github.com/samber/lo"
log "github.com/sirupsen/logrus"

"github.com/yugabyte/yb-voyager/yb-voyager/src/utils"
)

// Base parser interface
Expand Down Expand Up @@ -142,7 +144,11 @@ func (tableProcessor *TableProcessor) Process(parseTree *pg_query.ParseResult) (
}

func (tableProcessor *TableProcessor) parseTableElts(tableElts []*pg_query.Node, table *Table) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Few summary comments about table processor related code will help here.

/*
Parsing the table elements liek column definitions constraint basically all the things inside the () of CREATE TABLE test(id int, CONSTRAINT Pk PRIMARY KEY (id)....);
storing all the information of columns - name, typename, isArraytype and constraints - constraint name, columns involved, type of constraint, is deferrable or not

*/
for _, element := range tableElts {
if element.GetColumnDef() != nil {
if tableProcessor.isGeneratedColumn(element.GetColumnDef()) {
Expand Down Expand Up @@ -306,7 +312,7 @@ type TableColumn struct {
}

func (tc *TableColumn) GetFullTypeName() string {
return lo.Ternary(tc.TypeSchema != "", tc.TypeSchema+"."+tc.TypeName, tc.TypeName)
return utils.GetObjectName(tc.TypeSchema, tc.TypeName)
}

type TableConstraint struct {
Expand Down Expand Up @@ -340,8 +346,7 @@ func (c *TableConstraint) generateConstraintName(tableName string) string {
}

func (t *Table) GetObjectName() string {
qualifiedTable := lo.Ternary(t.SchemaName != "", fmt.Sprintf("%s.%s", t.SchemaName, t.TableName), t.TableName)
return qualifiedTable
return utils.GetObjectName(t.SchemaName, t.TableName)
}
func (t *Table) GetSchemaName() string { return t.SchemaName }

Expand Down Expand Up @@ -376,7 +381,7 @@ func (t *Table) addConstraint(conType pg_query.ConstrType, columns []string, spe
conName := lo.Ternary(specifiedConName == "", generatedConName, specifiedConName)
tc.ConstraintName = conName
if conType == FOREIGN_CONSTR_TYPE {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why doing this in case of foreign key only. Can this be a unconditional step?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

reference table is only present in case of foreign keys so using the field only in that case as it will be nil in other cases

tc.ReferencedTable = lo.Ternary(referencedTable.Schemaname != "", referencedTable.Schemaname+"."+referencedTable.Relname, referencedTable.Relname)
tc.ReferencedTable = utils.GetObjectName(referencedTable.Schemaname, referencedTable.Relname)
}
t.Constraints = append(t.Constraints, tc)
}
Expand Down Expand Up @@ -428,7 +433,7 @@ type ForeignTable struct {
}

func (f *ForeignTable) GetObjectName() string {
return lo.Ternary(f.SchemaName != "", f.SchemaName+"."+f.TableName, f.TableName)
return utils.GetObjectName(f.SchemaName, f.TableName)
}
func (f *ForeignTable) GetSchemaName() string { return f.SchemaName }

Expand Down Expand Up @@ -519,7 +524,7 @@ type IndexParam struct {
}

func (indexParam *IndexParam) GetFullExprCastTypeName() string {
return lo.Ternary(indexParam.ExprCastTypeSchema != "", indexParam.ExprCastTypeSchema+"."+indexParam.ExprCastTypeName, indexParam.ExprCastTypeName)
return utils.GetObjectName(indexParam.ExprCastTypeSchema, indexParam.ExprCastTypeName)
}

func (i *Index) GetObjectName() string {
Expand All @@ -528,7 +533,7 @@ func (i *Index) GetObjectName() string {
func (i *Index) GetSchemaName() string { return i.SchemaName }

func (i *Index) GetTableName() string {
return lo.Ternary(i.SchemaName != "", fmt.Sprintf("%s.%s", i.SchemaName, i.TableName), i.TableName)
return utils.GetObjectName(i.SchemaName, i.TableName)
}

func (i *Index) GetObjectType() string { return INDEX_OBJECT_TYPE }
Expand Down Expand Up @@ -606,7 +611,7 @@ func (atProcessor *AlterTableProcessor) Process(parseTree *pg_query.ParseResult)
In case of FKs the reference table is in PKTable field and columns are in FkAttrs
*/
alter.ConstraintColumns = parseColumnsFromKeys(constraint.FkAttrs)
alter.ConstraintReferencedTable = lo.Ternary(constraint.Pktable.Schemaname != "", constraint.Pktable.Schemaname+"."+constraint.Pktable.Relname, constraint.Pktable.Relname)
alter.ConstraintReferencedTable = utils.GetObjectName(constraint.Pktable.Schemaname, constraint.Pktable.Relname)
}

case pg_query.AlterTableType_AT_DisableRule:
Expand Down Expand Up @@ -646,8 +651,7 @@ type AlterTable struct {
}

func (a *AlterTable) GetObjectName() string {
qualifiedTable := lo.Ternary(a.SchemaName != "", fmt.Sprintf("%s.%s", a.SchemaName, a.TableName), a.TableName)
return qualifiedTable
return utils.GetObjectName(a.SchemaName, a.TableName)
}
func (a *AlterTable) GetSchemaName() string { return a.SchemaName }

Expand Down Expand Up @@ -712,7 +716,7 @@ type Policy struct {
}

func (p *Policy) GetObjectName() string {
qualifiedTable := lo.Ternary(p.SchemaName != "", fmt.Sprintf("%s.%s", p.SchemaName, p.TableName), p.TableName)
qualifiedTable := utils.GetObjectName(p.SchemaName, p.TableName)
return fmt.Sprintf("%s ON %s", p.PolicyName, qualifiedTable)
}
func (p *Policy) GetSchemaName() string { return p.SchemaName }
Expand Down Expand Up @@ -788,7 +792,7 @@ func (t *Trigger) GetObjectName() string {
}

func (t *Trigger) GetTableName() string {
return lo.Ternary(t.SchemaName != "", fmt.Sprintf("%s.%s", t.SchemaName, t.TableName), t.TableName)
return utils.GetObjectName(t.SchemaName, t.TableName)
}

func (t *Trigger) GetSchemaName() string { return t.SchemaName }
Expand Down Expand Up @@ -865,7 +869,7 @@ type CreateType struct {
}

func (c *CreateType) GetObjectName() string {
return lo.Ternary(c.SchemaName != "", fmt.Sprintf("%s.%s", c.SchemaName, c.TypeName), c.TypeName)
return utils.GetObjectName(c.SchemaName, c.TypeName)
}
func (c *CreateType) GetSchemaName() string { return c.SchemaName }

Expand All @@ -887,7 +891,7 @@ func (v *ViewProcessor) Process(parseTree *pg_query.ParseResult) (DDLObject, err

viewSchemaName := viewNode.ViewStmt.View.Schemaname
viewName := viewNode.ViewStmt.View.Relname
qualifiedViewName := lo.Ternary(viewSchemaName == "", viewName, viewSchemaName+"."+viewName)
qualifiedViewName := utils.GetObjectName(viewSchemaName, viewName)

/*
view_stmt:{view:{schemaname:"public" relname:"invoker_view" inh:true relpersistence:"p" location:12}
Expand Down Expand Up @@ -920,7 +924,7 @@ type View struct {
}

func (v *View) GetObjectName() string {
return lo.Ternary(v.SchemaName != "", fmt.Sprintf("%s.%s", v.SchemaName, v.ViewName), v.ViewName)
return utils.GetObjectName(v.SchemaName, v.ViewName)
}
func (v *View) GetSchemaName() string { return v.SchemaName }

Expand Down Expand Up @@ -952,7 +956,7 @@ type MView struct {
}

func (mv *MView) GetObjectName() string {
return lo.Ternary(mv.SchemaName != "", fmt.Sprintf("%s.%s", mv.SchemaName, mv.ViewName), mv.ViewName)
return utils.GetObjectName(mv.SchemaName, mv.ViewName)
}
func (mv *MView) GetSchemaName() string { return mv.SchemaName }

Expand Down
12 changes: 0 additions & 12 deletions yb-voyager/src/query/queryparser/helpers_protomsg.go
Original file line number Diff line number Diff line change
Expand Up @@ -420,18 +420,6 @@ func ProtoAsSelectStmt(msg protoreflect.Message) (*pg_query.SelectStmt, error) {
return selectStmtNode, nil
}

func GetConstraintNode(msg protoreflect.Message) (*pg_query.Constraint, error) {
protoMsg, ok := msg.Interface().(proto.Message)
if !ok {
return nil, fmt.Errorf("failed to cast msg to proto.Message")
}
constNode, ok := protoMsg.(*pg_query.Constraint)
if !ok {
return nil, fmt.Errorf("failed to cast msg to %s", PG_QUERY_CONSTRAINT_NODE)
}
return constNode, nil
}

/*
Example:
options:{def_elem:{defname:"security_invoker" arg:{string:{sval:"true"}} defaction:DEFELEM_UNSPEC location:32}}
Expand Down
9 changes: 5 additions & 4 deletions yb-voyager/src/query/queryparser/helpers_struct.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ import (
"strings"

pg_query "github.com/pganalyze/pg_query_go/v6"
"github.com/samber/lo"

"github.com/yugabyte/yb-voyager/yb-voyager/src/utils"
)

const (
Expand Down Expand Up @@ -68,7 +69,7 @@ func GetObjectTypeAndObjectName(parseTree *pg_query.ParseResult) (string, string
}
funcNameList := stmt.GetFuncname()
funcSchemaName, funcName := getFunctionObjectName(funcNameList)
return objectType, lo.Ternary(funcSchemaName != "", fmt.Sprintf("%s.%s", funcSchemaName, funcName), funcName)
return objectType, utils.GetObjectName(funcSchemaName, funcName)
case isViewStmt:
viewName := viewNode.ViewStmt.View
return "VIEW", getObjectNameFromRangeVar(viewName)
Expand All @@ -83,7 +84,7 @@ func GetObjectTypeAndObjectName(parseTree *pg_query.ParseResult) (string, string
indexName := createIndexNode.IndexStmt.Idxname
schemaName := createIndexNode.IndexStmt.Relation.GetSchemaname()
tableName := createIndexNode.IndexStmt.Relation.GetRelname()
fullyQualifiedName := lo.Ternary(schemaName != "", schemaName+"."+tableName, tableName)
fullyQualifiedName := utils.GetObjectName(schemaName, tableName)
displayObjName := fmt.Sprintf("%s ON %s", indexName, fullyQualifiedName)
return "INDEX", displayObjName
default:
Expand All @@ -99,7 +100,7 @@ func isArrayType(typeName *pg_query.TypeName) bool {
func getObjectNameFromRangeVar(obj *pg_query.RangeVar) string {
schema := obj.Schemaname
name := obj.Relname
return lo.Ternary(schema != "", fmt.Sprintf("%s.%s", schema, name), name)
return utils.GetObjectName(schema, name)
}

func getFunctionObjectName(funcNameList []*pg_query.Node) (string, string) {
Expand Down
10 changes: 6 additions & 4 deletions yb-voyager/src/query/queryparser/object_collector.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,10 @@ import (

"github.com/samber/lo"
log "github.com/sirupsen/logrus"
"github.com/yugabyte/yb-voyager/yb-voyager/src/constants"
"google.golang.org/protobuf/reflect/protoreflect"

"github.com/yugabyte/yb-voyager/yb-voyager/src/constants"
"github.com/yugabyte/yb-voyager/yb-voyager/src/utils"
)

// ObjectPredicate defines a function signature that decides whether to include an object.
Expand Down Expand Up @@ -76,7 +78,7 @@ func (c *ObjectCollector) Collect(msg protoreflect.Message) {
case PG_QUERY_RANGEVAR_NODE:
schemaName := GetStringField(msg, "schemaname")
relName := GetStringField(msg, "relname")
objectName := lo.Ternary(schemaName != "", schemaName+"."+relName, relName)
objectName := utils.GetObjectName(schemaName, relName)
log.Debugf("[RangeVar] fetched schemaname=%s relname=%s objectname=%s field\n", schemaName, relName, objectName)
// it will be either table or view, considering objectType=table for both
if c.predicate(schemaName, relName, constants.TABLE) {
Expand All @@ -89,7 +91,7 @@ func (c *ObjectCollector) Collect(msg protoreflect.Message) {
if relationMsg != nil {
schemaName := GetStringField(relationMsg, "schemaname")
relName := GetStringField(relationMsg, "relname")
objectName := lo.Ternary(schemaName != "", schemaName+"."+relName, relName)
objectName := utils.GetObjectName(schemaName, relName)
log.Debugf("[IUD] fetched schemaname=%s relname=%s objectname=%s field\n", schemaName, relName, objectName)
if c.predicate(schemaName, relName, "table") {
c.addObject(objectName)
Expand All @@ -103,7 +105,7 @@ func (c *ObjectCollector) Collect(msg protoreflect.Message) {
return
}

objectName := lo.Ternary(schemaName != "", schemaName+"."+functionName, functionName)
objectName := utils.GetObjectName(schemaName, functionName)
log.Debugf("[Funccall] fetched schemaname=%s objectname=%s field\n", schemaName, objectName)
if c.predicate(schemaName, functionName, constants.FUNCTION) {
c.addObject(objectName)
Expand Down
4 changes: 4 additions & 0 deletions yb-voyager/src/utils/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -740,3 +740,7 @@ func CheckTools(tools ...string) []string {

return missingTools
}

func GetObjectName(schemaName, objName string) string {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: BuildObjectName

return lo.Ternary(schemaName != "", schemaName+"."+objName, objName)
}